notebooks/2024_01_26_Evaluation_Kurs_V2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1yaIim5phwaRgzIp9TSloyHU5tsxG-KbS",
      "authorship_tag": "ABX9TyN8cQkqPcs9bVJKokAOgPtu",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5eb32be558644515b5709afb16483592": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b99de8e04d6e407b8ff20f58ff6c2276",
              "IPY_MODEL_87eeaaade0e443d7b76caae8bf4165b2",
              "IPY_MODEL_a1afaf96d68e498897e03c2bde9aab55"
            ],
            "layout": "IPY_MODEL_c6e5f48a00b040b88e2fa5619af8c8cb"
          }
        },
        "b99de8e04d6e407b8ff20f58ff6c2276": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fa41010456da4d508b9c3950dc54638a",
            "placeholder": "​",
            "style": "IPY_MODEL_4556624f102745f88ce41e8ec7dfd9d9",
            "value": "100%"
          }
        },
        "87eeaaade0e443d7b76caae8bf4165b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c341ef0f630c40b691f3c9cc31b39f25",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f6472c6b75ab4bdda77e99a2cf62e9e5",
            "value": 3
          }
        },
        "a1afaf96d68e498897e03c2bde9aab55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1fb53009dd7c4893945b69e0e8293dee",
            "placeholder": "​",
            "style": "IPY_MODEL_be683eafd3c946beb86d6529105120b4",
            "value": " 3/3 [00:00&lt;00:00, 31.16it/s]"
          }
        },
        "c6e5f48a00b040b88e2fa5619af8c8cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa41010456da4d508b9c3950dc54638a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4556624f102745f88ce41e8ec7dfd9d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c341ef0f630c40b691f3c9cc31b39f25": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f6472c6b75ab4bdda77e99a2cf62e9e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1fb53009dd7c4893945b69e0e8293dee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be683eafd3c946beb86d6529105120b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a052c5854c004aa48c7d38208916bcab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f11347431bed42a0b60b37a3cfc09731",
              "IPY_MODEL_109f754d039f44cba793e3a100bb16ad",
              "IPY_MODEL_d060f3857cdd419e8ea13e1a12432045"
            ],
            "layout": "IPY_MODEL_3685c97ef4d141ba9e44496364963d6c"
          }
        },
        "f11347431bed42a0b60b37a3cfc09731": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f02ade2608f44e4a9f08be789edaf48d",
            "placeholder": "​",
            "style": "IPY_MODEL_93df435bb767440c93fa3860b8fe41f9",
            "value": "100%"
          }
        },
        "109f754d039f44cba793e3a100bb16ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5b681d0d9f8f453caf7af958c59c7991",
            "max": 447,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_82105eeb117944c0a0ebd60cd44c74e1",
            "value": 447
          }
        },
        "d060f3857cdd419e8ea13e1a12432045": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ff2fd1b67cc94606b1620fe2911a69cf",
            "placeholder": "​",
            "style": "IPY_MODEL_f1dc3efbd78545f086ead5300ae34464",
            "value": " 447/447 [00:00&lt;00:00, 18021.72it/s]"
          }
        },
        "3685c97ef4d141ba9e44496364963d6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f02ade2608f44e4a9f08be789edaf48d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93df435bb767440c93fa3860b8fe41f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5b681d0d9f8f453caf7af958c59c7991": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82105eeb117944c0a0ebd60cd44c74e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ff2fd1b67cc94606b1620fe2911a69cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f1dc3efbd78545f086ead5300ae34464": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "67804cc5d7784aad9e0b691cab630d0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5586ad2cb3f64081bfe96e08eaa344cd",
              "IPY_MODEL_fd1f618f84f647769e7490cfaaac2016",
              "IPY_MODEL_156e59b3c9b14b1b89464d5bd7800811"
            ],
            "layout": "IPY_MODEL_fd3f94592cc042e6826a31aeb68ad079"
          }
        },
        "5586ad2cb3f64081bfe96e08eaa344cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_abaa6eccffa7450985fd2882004c55e0",
            "placeholder": "​",
            "style": "IPY_MODEL_f6bc4188551646c3b5e436c9eb4a269b",
            "value": "100%"
          }
        },
        "fd1f618f84f647769e7490cfaaac2016": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd9dadbb073249e88c531b2f0eadf5ef",
            "max": 447,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7722908c8cbb4104adec8e518afa33b1",
            "value": 447
          }
        },
        "156e59b3c9b14b1b89464d5bd7800811": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ca9d4cc9e3db4289ac33514548983166",
            "placeholder": "​",
            "style": "IPY_MODEL_795ca0821a894768ab993d15ccc58b47",
            "value": " 447/447 [00:00&lt;00:00, 17660.14it/s]"
          }
        },
        "fd3f94592cc042e6826a31aeb68ad079": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "abaa6eccffa7450985fd2882004c55e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f6bc4188551646c3b5e436c9eb4a269b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cd9dadbb073249e88c531b2f0eadf5ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7722908c8cbb4104adec8e518afa33b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ca9d4cc9e3db4289ac33514548983166": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "795ca0821a894768ab993d15ccc58b47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "26102edfd83c4a8d86af1a5e537d0f8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5f2806587de241a8aa590fdb8a3a180c",
              "IPY_MODEL_7c6df0b9955c46e48a7eec5c9d1881e7",
              "IPY_MODEL_1ca5a4b6314644d298c2c652108be1fe"
            ],
            "layout": "IPY_MODEL_0f9e02b822014e589d099bf09e00b8bb"
          }
        },
        "5f2806587de241a8aa590fdb8a3a180c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c48318d3aa134b7380dea001907a5c89",
            "placeholder": "​",
            "style": "IPY_MODEL_3f3a4007e71b4975b77bfd25c2b515c4",
            "value": "100%"
          }
        },
        "7c6df0b9955c46e48a7eec5c9d1881e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_608ca438db7341f6a710212b87223def",
            "max": 446,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b78246667b554d799181c6fb74b3d22f",
            "value": 446
          }
        },
        "1ca5a4b6314644d298c2c652108be1fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f90c0bd878d04647a9b5536bb4e35ac3",
            "placeholder": "​",
            "style": "IPY_MODEL_a85ade2311374c4cbb430e8b9c95c7f3",
            "value": " 446/446 [00:00&lt;00:00, 18636.89it/s]"
          }
        },
        "0f9e02b822014e589d099bf09e00b8bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c48318d3aa134b7380dea001907a5c89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f3a4007e71b4975b77bfd25c2b515c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "608ca438db7341f6a710212b87223def": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b78246667b554d799181c6fb74b3d22f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f90c0bd878d04647a9b5536bb4e35ac3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a85ade2311374c4cbb430e8b9c95c7f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/michaelachmann/social-media-lab/blob/main/notebooks/2024_01_26_Evaluation_Kurs_V2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# GPT Evaluation Notebook V2 [![DOI](https://zenodo.org/badge/660157642.svg)](https://zenodo.org/badge/latestdoi/660157642)\n",
        "![Notes on (Computational) Social Media Research Banner](https://raw.githubusercontent.com/michaelachmann/social-media-lab/main/images/banner.png)\n",
        "\n",
        "## Overview\n",
        "\n",
        "This Jupyter notebook is a part of the social-media-lab.net project, which is a work-in-progress textbook on computational social media analysis. The notebook is intended for use in my classes.\n",
        "\n",
        "The **GPT Evaluation V2** Notebook helps to calculate interrater agreement scores like Cohen's Kappa and Krippendorff's Alpha for a combination of human annotations (exported from [Label Studio](https://labelstud.io/) and GPT- (or other ML/LLM) based annotations. Version two handles multiple JSON files as input, compatible with the open source version of LabelStudio.\n",
        "\n",
        "### Project Information\n",
        "\n",
        "- Project Website: [social-media-lab.net](https://social-media-lab.net/)\n",
        "- GitHub Repository: [https://github.com/michaelachmann/social-media-lab](https://github.com/michaelachmann/social-media-lab)\n",
        "\n",
        "## License Information\n",
        "\n",
        "This notebook, along with all other notebooks in the project, is licensed under the following terms:\n",
        "\n",
        "- License: [GNU General Public License version 3.0 (GPL-3.0)](https://www.gnu.org/licenses/gpl-3.0.de.html)\n",
        "- License File: [LICENSE.md](https://github.com/michaelachmann/social-media-lab/blob/main/LICENSE.md)\n",
        "\n",
        "\n",
        "## Citation\n",
        "\n",
        "If you use or reference this notebook in your work, please cite it appropriately. Here is an example of the citation:\n",
        "\n",
        "```\n",
        "Michael Achmann. (2024). michaelachmann/social-media-lab: 2024-01-29 (v0.0.11). Zenodo. https://doi.org/10.5281/zenodo.8199902\n",
        "```"
      ],
      "metadata": {
        "id": "5VzE5KfwA3lN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "5eb32be558644515b5709afb16483592",
            "b99de8e04d6e407b8ff20f58ff6c2276",
            "87eeaaade0e443d7b76caae8bf4165b2",
            "a1afaf96d68e498897e03c2bde9aab55",
            "c6e5f48a00b040b88e2fa5619af8c8cb",
            "fa41010456da4d508b9c3950dc54638a",
            "4556624f102745f88ce41e8ec7dfd9d9",
            "c341ef0f630c40b691f3c9cc31b39f25",
            "f6472c6b75ab4bdda77e99a2cf62e9e5",
            "1fb53009dd7c4893945b69e0e8293dee",
            "be683eafd3c946beb86d6529105120b4",
            "a052c5854c004aa48c7d38208916bcab",
            "f11347431bed42a0b60b37a3cfc09731",
            "109f754d039f44cba793e3a100bb16ad",
            "d060f3857cdd419e8ea13e1a12432045",
            "3685c97ef4d141ba9e44496364963d6c",
            "f02ade2608f44e4a9f08be789edaf48d",
            "93df435bb767440c93fa3860b8fe41f9",
            "5b681d0d9f8f453caf7af958c59c7991",
            "82105eeb117944c0a0ebd60cd44c74e1",
            "ff2fd1b67cc94606b1620fe2911a69cf",
            "f1dc3efbd78545f086ead5300ae34464",
            "67804cc5d7784aad9e0b691cab630d0d",
            "5586ad2cb3f64081bfe96e08eaa344cd",
            "fd1f618f84f647769e7490cfaaac2016",
            "156e59b3c9b14b1b89464d5bd7800811",
            "fd3f94592cc042e6826a31aeb68ad079",
            "abaa6eccffa7450985fd2882004c55e0",
            "f6bc4188551646c3b5e436c9eb4a269b",
            "cd9dadbb073249e88c531b2f0eadf5ef",
            "7722908c8cbb4104adec8e518afa33b1",
            "ca9d4cc9e3db4289ac33514548983166",
            "795ca0821a894768ab993d15ccc58b47",
            "26102edfd83c4a8d86af1a5e537d0f8e",
            "5f2806587de241a8aa590fdb8a3a180c",
            "7c6df0b9955c46e48a7eec5c9d1881e7",
            "1ca5a4b6314644d298c2c652108be1fe",
            "0f9e02b822014e589d099bf09e00b8bb",
            "c48318d3aa134b7380dea001907a5c89",
            "3f3a4007e71b4975b77bfd25c2b515c4",
            "608ca438db7341f6a710212b87223def",
            "b78246667b554d799181c6fb74b3d22f",
            "f90c0bd878d04647a9b5536bb4e35ac3",
            "a85ade2311374c4cbb430e8b9c95c7f3"
          ]
        },
        "cellView": "form",
        "id": "V-rd9LTEAFn3",
        "outputId": "b8ab0300-56ef-4f01-c2aa-b48432c7cc0e"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5eb32be558644515b5709afb16483592"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/447 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a052c5854c004aa48c7d38208916bcab"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/447 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "67804cc5d7784aad9e0b691cab630d0d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/446 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "26102edfd83c4a8d86af1a5e537d0f8e"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "# @title Read the LabelStudio Annotations from file\n",
        "# @markdown <p><b>NEW</b> This version deals with multiple annotation projects. The community version of Label Studio does not handle more than one annotator per item. Setup three different projects using the same data and have one annotator use one project. Export the results and add the files to the list below. The python loop creates one annotation dataset from the three annotations projects and provides all interrater reliability measures as with previous versions of this notebook.<p>\n",
        "# @markdown <p><b>NEW</b> This version handles `labels` for the annotation of bounding boxes in images.</p>\n",
        "# @markdown <p><b>NOTE</b> This version does not handle reviews!</p>\n",
        "\n",
        "human_annotations_jsons = ['/content/drive/MyDrive/2023-12-22-BA-Pirkl/political-stories-1-annotated.json',\n",
        "                           '/content/drive/MyDrive/2023-12-22-BA-Pirkl/political-stories-2-annotated.json',\n",
        "                           '/content/drive/MyDrive/2023-12-22-BA-Pirkl/political-stories-4-annotated.json']\n",
        "\n",
        "import json\n",
        "import pandas as pd\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "exported_annotations = []\n",
        "\n",
        "for human_annotations_json in human_annotations_jsons:\n",
        "  with open(human_annotations_json, 'r') as f:\n",
        "    j = f.read()\n",
        "    exported_annotations.append(json.loads(j))\n",
        "\n",
        "\n",
        "\n",
        "def process_result(result, coder, md):\n",
        "    value_type = result.get('type', \"\")\n",
        "    metadata = {\n",
        "        **md,\n",
        "        \"coder\": coder,\n",
        "        \"from_name\": result.get('from_name', \"\")\n",
        "    }\n",
        "    annotations = []\n",
        "    if value_type == \"choices\":\n",
        "        choices = result['value'].get('choices', [])\n",
        "        for choice in choices:\n",
        "            r = {**metadata}\n",
        "            r['value'] = choice\n",
        "            annotations.append(r)\n",
        "    elif value_type == \"taxonomy\":\n",
        "        taxonomies = result['value'].get('taxonomy', [])\n",
        "        for taxonomy in taxonomies:\n",
        "            if len(taxonomy) > 1:\n",
        "                taxonomy = \" > \".join(taxonomy)\n",
        "            elif len(taxonomy) == 1:\n",
        "                taxonomy = taxonomy[0]\n",
        "            r = {**metadata}\n",
        "            r['value'] = taxonomy\n",
        "            annotations.append(r)\n",
        "    elif value_type == \"labels\":\n",
        "        labels = result['value'].get('labels', [])\n",
        "        labels_id = result.get('id', '')\n",
        "        label = \"\".join(labels)\n",
        "        r = {**metadata}\n",
        "        r['value'] = label\n",
        "        r['id'] = labels_id\n",
        "        annotations.append(r)\n",
        "\n",
        "    return annotations\n",
        "\n",
        "all_annotations = []\n",
        "for exported_annotation in tqdm(exported_annotations):\n",
        "  for data in tqdm(exported_annotation):\n",
        "      annotations = data.get(\"annotations\")\n",
        "      metadata = {\n",
        "          **data.get(\"data\")\n",
        "      }\n",
        "\n",
        "      for annotation in annotations:\n",
        "        coder = annotation['completed_by']\n",
        "        results = annotation.get(\"result\")\n",
        "        if results:\n",
        "          for result in results:\n",
        "            all_annotations.extend(process_result(result, coder, metadata))\n",
        "        else:\n",
        "          print(\"Skipped Missing Result\")\n",
        "\n",
        "\n",
        "all_annotations_df = pd.DataFrame(all_annotations)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the dataframe\n",
        "all_annotations_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "afJj59jrNsa-",
        "outputId": "385109c2-fb49-4e9d-f427-8ebdc651f99a"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                               image  coder from_name  \\\n",
              "0  gs://ig-politik-stories/armin_laschet_26-09-20...  11926    labels   \n",
              "1  gs://ig-politik-stories/armin_laschet_26-09-20...  11478    labels   \n",
              "2  gs://ig-politik-stories/spdde_26-09-2021_00:00...  11926    labels   \n",
              "3  gs://ig-politik-stories/spdde_26-09-2021_00:00...  11478    labels   \n",
              "4  gs://ig-politik-stories/armin_laschet_20-09-20...  11926    labels   \n",
              "\n",
              "           value                                    id  \n",
              "0  Armin Laschet  0b8b3aa1-e602-4d45-a088-e7f3b57e2306  \n",
              "1  Armin Laschet  0b8b3aa1-e602-4d45-a088-e7f3b57e2306  \n",
              "2    Olaf Scholz  523fccf4-70e1-437e-bb1d-c906fd0d4648  \n",
              "3    Olaf Scholz  523fccf4-70e1-437e-bb1d-c906fd0d4648  \n",
              "4  Armin Laschet  b8205fa2-9302-4ba6-82ff-00587434e0f9  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2bfa19c5-6db5-447f-b12b-ebdf04b7a25e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>image</th>\n",
              "      <th>coder</th>\n",
              "      <th>from_name</th>\n",
              "      <th>value</th>\n",
              "      <th>id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>gs://ig-politik-stories/armin_laschet_26-09-20...</td>\n",
              "      <td>11926</td>\n",
              "      <td>labels</td>\n",
              "      <td>Armin Laschet</td>\n",
              "      <td>0b8b3aa1-e602-4d45-a088-e7f3b57e2306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>gs://ig-politik-stories/armin_laschet_26-09-20...</td>\n",
              "      <td>11478</td>\n",
              "      <td>labels</td>\n",
              "      <td>Armin Laschet</td>\n",
              "      <td>0b8b3aa1-e602-4d45-a088-e7f3b57e2306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>gs://ig-politik-stories/spdde_26-09-2021_00:00...</td>\n",
              "      <td>11926</td>\n",
              "      <td>labels</td>\n",
              "      <td>Olaf Scholz</td>\n",
              "      <td>523fccf4-70e1-437e-bb1d-c906fd0d4648</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>gs://ig-politik-stories/spdde_26-09-2021_00:00...</td>\n",
              "      <td>11478</td>\n",
              "      <td>labels</td>\n",
              "      <td>Olaf Scholz</td>\n",
              "      <td>523fccf4-70e1-437e-bb1d-c906fd0d4648</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>gs://ig-politik-stories/armin_laschet_20-09-20...</td>\n",
              "      <td>11926</td>\n",
              "      <td>labels</td>\n",
              "      <td>Armin Laschet</td>\n",
              "      <td>b8205fa2-9302-4ba6-82ff-00587434e0f9</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2bfa19c5-6db5-447f-b12b-ebdf04b7a25e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2bfa19c5-6db5-447f-b12b-ebdf04b7a25e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2bfa19c5-6db5-447f-b12b-ebdf04b7a25e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-f4cba944-5dcb-4edf-bb7d-46bdd34c5b48\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f4cba944-5dcb-4edf-bb7d-46bdd34c5b48')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-f4cba944-5dcb-4edf-bb7d-46bdd34c5b48 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Contingency Table\n",
        "\n",
        "We select one coding in this step (e.g. Positioning) and create a contingency table where each annotated item (text, image) occupies one row and each coder one column. The `identifier` should be set to a unique column, like `uuid` or `Filename`."
      ],
      "metadata": {
        "id": "ah-wpvKaFWQX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @markdown  Enter the `from_name` for the variable you're interested in at the moment. (Refer to your LabelStudio Interface for the right `from_name`).\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from_name = 'labels' # @param {type:\"string\"}\n",
        "identifier = 'id' # @param {type:\"string\"}\n",
        "\n",
        "# Assuming your DataFrame is named 'df'\n",
        "filtered_df = all_annotations_df[all_annotations_df['from_name'].str.contains(from_name, case=False)]\n",
        "\n",
        "def to_bool(val):\n",
        "  if isinstance(val, bool):\n",
        "    return val\n",
        "  if isinstance(val, str):\n",
        "    return val.lower() == \"true\"\n",
        "\n",
        "  return bool(val)\n",
        "\n",
        "values = filtered_df['value'].unique()\n",
        "identifier_values = filtered_df[identifier].unique()\n",
        "\n",
        "contingency_matrix = pd.crosstab(filtered_df[identifier], filtered_df['coder'], values=filtered_df['value'], aggfunc='first')\n",
        "contingency_matrix = contingency_matrix.reindex(identifier_values)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "u7dfytyfFV_M"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's take a look at the contigency table. We refer to each coder using a pseudonymous number.\n",
        "contingency_matrix.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "4RGzP0UeGLJm",
        "outputId": "ce7bd9f6-8713-4245-b5a2-cd83276693b2"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "coder                                9368               11478 11766 11802  \\\n",
              "id                                                                          \n",
              "0b8b3aa1-e602-4d45-a088-e7f3b57e2306   NaN      Armin Laschet   NaN   NaN   \n",
              "523fccf4-70e1-437e-bb1d-c906fd0d4648   NaN        Olaf Scholz   NaN   NaN   \n",
              "b8205fa2-9302-4ba6-82ff-00587434e0f9   NaN      Armin Laschet   NaN   NaN   \n",
              "a6c63259-e37d-4099-b55e-095b62e64bcd   NaN        Nicola Beer   NaN   NaN   \n",
              "a44eed36-cfa8-4187-b3f4-af8ccf7c5241   NaN  Christian Lindner   NaN   NaN   \n",
              "\n",
              "coder                                11909              11926  \n",
              "id                                                             \n",
              "0b8b3aa1-e602-4d45-a088-e7f3b57e2306   NaN      Armin Laschet  \n",
              "523fccf4-70e1-437e-bb1d-c906fd0d4648   NaN        Olaf Scholz  \n",
              "b8205fa2-9302-4ba6-82ff-00587434e0f9   NaN      Armin Laschet  \n",
              "a6c63259-e37d-4099-b55e-095b62e64bcd   NaN        Nicola Beer  \n",
              "a44eed36-cfa8-4187-b3f4-af8ccf7c5241   NaN  Christian Lindner  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-81bd1783-1dc9-483f-b4cc-99a44fb5e0dc\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>coder</th>\n",
              "      <th>9368</th>\n",
              "      <th>11478</th>\n",
              "      <th>11766</th>\n",
              "      <th>11802</th>\n",
              "      <th>11909</th>\n",
              "      <th>11926</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0b8b3aa1-e602-4d45-a088-e7f3b57e2306</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Armin Laschet</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Armin Laschet</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>523fccf4-70e1-437e-bb1d-c906fd0d4648</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Olaf Scholz</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Olaf Scholz</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b8205fa2-9302-4ba6-82ff-00587434e0f9</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Armin Laschet</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Armin Laschet</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>a6c63259-e37d-4099-b55e-095b62e64bcd</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Nicola Beer</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Nicola Beer</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>a44eed36-cfa8-4187-b3f4-af8ccf7c5241</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Christian Lindner</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Christian Lindner</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-81bd1783-1dc9-483f-b4cc-99a44fb5e0dc')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-81bd1783-1dc9-483f-b4cc-99a44fb5e0dc button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-81bd1783-1dc9-483f-b4cc-99a44fb5e0dc');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c3880024-61fa-4db1-af6a-dccc674ad632\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c3880024-61fa-4db1-af6a-dccc674ad632')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c3880024-61fa-4db1-af6a-dccc674ad632 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Calculate Pairwise $\\kappa$\n",
        "\n",
        "Let's calculate Cohen's Kappa for each coder pair."
      ],
      "metadata": {
        "id": "R8B5QeTxHuRF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import cohen_kappa_score\n",
        "\n",
        "# Function to calculate Cohen's Kappa for each pair of raters\n",
        "def calculate_kappa(matrix):\n",
        "    raters = matrix.columns\n",
        "    kappa_scores = []\n",
        "    for i in range(len(raters)):\n",
        "        for j in range(i+1, len(raters)):\n",
        "            rater1, rater2 = raters[i], raters[j]\n",
        "            # Drop NA values for the pair of raters\n",
        "            pair_matrix = matrix[[rater1, rater2]].dropna()\n",
        "            # Skip pairs without overlaps\n",
        "            if len(pair_matrix) > 0:\n",
        "              kappa = cohen_kappa_score(pair_matrix[rater1], pair_matrix[rater2])\n",
        "              kappa_scores.append({\n",
        "                  \"Coder 1\": rater1,\n",
        "                  \"Coder 2\": rater2,\n",
        "                  \"Kappa\": kappa,\n",
        "                  \"Overlap\": len(pair_matrix),\n",
        "                  \"Coding\": from_name\n",
        "                  })\n",
        "    return kappa_scores\n",
        "\n",
        "kappa_scores = calculate_kappa(contingency_matrix)\n",
        "kappa_scores_df = pd.DataFrame(kappa_scores)"
      ],
      "metadata": {
        "id": "yFoOTkxfG78n"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's display the pairwise Kappa agreements.\n",
        "kappa_scores_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "6JtZrIg_ILTu",
        "outputId": "28097a67-3627-4de3-8cd8-d07fc392d092"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Coder 1  Coder 2     Kappa  Overlap  Coding\n",
              "0     9368    11766  0.977423      213  labels\n",
              "1    11478    11926  0.986464      258  labels\n",
              "2    11802    11909  0.972419      258  labels"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f73a2a8c-674a-4413-8d38-f6d852f7ed03\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Coder 1</th>\n",
              "      <th>Coder 2</th>\n",
              "      <th>Kappa</th>\n",
              "      <th>Overlap</th>\n",
              "      <th>Coding</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>9368</td>\n",
              "      <td>11766</td>\n",
              "      <td>0.977423</td>\n",
              "      <td>213</td>\n",
              "      <td>labels</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>11478</td>\n",
              "      <td>11926</td>\n",
              "      <td>0.986464</td>\n",
              "      <td>258</td>\n",
              "      <td>labels</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>11802</td>\n",
              "      <td>11909</td>\n",
              "      <td>0.972419</td>\n",
              "      <td>258</td>\n",
              "      <td>labels</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f73a2a8c-674a-4413-8d38-f6d852f7ed03')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-f73a2a8c-674a-4413-8d38-f6d852f7ed03 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-f73a2a8c-674a-4413-8d38-f6d852f7ed03');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d27786e2-7eac-43b2-b75e-ea58636ad0de\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d27786e2-7eac-43b2-b75e-ea58636ad0de')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d27786e2-7eac-43b2-b75e-ea58636ad0de button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Overall Agreement: Krippendorff's $\\alpha$\n",
        "\n",
        "Next, we calculate the overall agreement using Krippendorffs Alpha. First we need to install the package."
      ],
      "metadata": {
        "id": "5_j_OoMPJNQo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install krippendorff"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ab2nLCtlJZPb",
        "outputId": "11634e46-1b04-4783-bd8a-7871c964846a"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting krippendorff\n",
            "  Downloading krippendorff-0.6.1-py3-none-any.whl (18 kB)\n",
            "Requirement already satisfied: numpy<2.0,>=1.21 in /usr/local/lib/python3.10/dist-packages (from krippendorff) (1.23.5)\n",
            "Installing collected packages: krippendorff\n",
            "Successfully installed krippendorff-0.6.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import krippendorff\n",
        "import pandas as pd\n",
        "\n",
        "def convert_to_reliability_data(matrix):\n",
        "    transposed_matrix = matrix.T\n",
        "    reliability_data = []\n",
        "    for _, ratings in transposed_matrix.iterrows():\n",
        "        reliability_data.append(ratings.tolist())\n",
        "    return reliability_data\n",
        "\n",
        "reliability_data = convert_to_reliability_data(contingency_matrix)\n",
        "\n",
        "# Calculating Krippendorff's Alpha treating \"Unsure\" as a distinct category\n",
        "alpha = krippendorff.alpha(reliability_data=reliability_data, level_of_measurement='nominal')\n",
        "\n",
        "print(\"Krippendorff's Alpha:\", alpha)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xeLbgwPXGZo0",
        "outputId": "62fbaf91-62ec-4737-f434-1c7c2128f6e5"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Krippendorff's Alpha: 0.9790399150132238\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Majority Decision\n",
        "\n",
        "One approach to assess the quality of machine labelled data is the comparison between machine-generated labels and human-generated labels, commonly known as \"gold standard\" labels. This process is called \"label agreement\" or \"inter-rater agreement\" and is widely used in various fields, including natural language processing, machine learning, and computational social science.\n",
        "\n",
        "We are going to use create a `majority_decision` column using the human annotations: We have chosen an uneven number of annotators in order to find a majority for each label. First, we are going to create a contingency table (or matrix), then we can determine the majority decision."
      ],
      "metadata": {
        "id": "T9gGSnGSNx7q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# Each row represents an item, and each column a decision from a different annotator.\n",
        "\n",
        "# Step 1: Find the mode (most common decision) for each row\n",
        "# The mode is used as it represents the majority decision.\n",
        "# decisions will have the most frequent value in each row, handling ties by keeping all modes.\n",
        "decisions = contingency_matrix.mode(axis=1)\n",
        "\n",
        "# Step 2: Extract the primary mode (first column after mode operation)\n",
        "# This represents the majority decision. If there's a tie, it takes the first one.\n",
        "majority_decisions = decisions.iloc[:, 0]\n",
        "\n",
        "##########\n",
        "## Warning: This part needs some refactoring. Will be updated shortly.\n",
        "##########\n",
        "# Step 3: Count the number of non-NaN values (actual decisions) per row, excluding the first column\n",
        "# Use .iloc[:, 1:] to exclude the first column\n",
        "# row_non_nan_counts = contingency_matrix.iloc[:, 1:].notnull().sum(axis=1)\n",
        "\n",
        "# Step 4: Invalidate the majority decision where the number of decisions is insufficient or even\n",
        "# Majority decisions are only considered valid if there are more than 2 decisions and the number of decisions is odd.\n",
        "\n",
        "# Define a condition for invalidating rows\n",
        "# invalid_rows_condition = (row_non_nan_counts < 3) | (row_non_nan_counts % 2 == 0)\n",
        "\n",
        "\n",
        "# Step 5: Append the majority decision as a new column in the contingency matrix\n",
        "# This column now represents the aggregated decision from the annotators per item.\n",
        "contingency_matrix['Majority Decision'] = majority_decisions"
      ],
      "metadata": {
        "id": "V6ty8wugN7OI"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "contingency_matrix.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "qsECeqg8wzjU",
        "outputId": "fff181da-0470-4adb-aa79-cd0e0120d922"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "coder                                9368              11478 11766 11802  \\\n",
              "id                                                                         \n",
              "0b8b3aa1-e602-4d45-a088-e7f3b57e2306  NaN      Armin Laschet   NaN   NaN   \n",
              "523fccf4-70e1-437e-bb1d-c906fd0d4648  NaN        Olaf Scholz   NaN   NaN   \n",
              "b8205fa2-9302-4ba6-82ff-00587434e0f9  NaN      Armin Laschet   NaN   NaN   \n",
              "a6c63259-e37d-4099-b55e-095b62e64bcd  NaN        Nicola Beer   NaN   NaN   \n",
              "a44eed36-cfa8-4187-b3f4-af8ccf7c5241  NaN  Christian Lindner   NaN   NaN   \n",
              "\n",
              "coder                                11909              11926  \\\n",
              "id                                                              \n",
              "0b8b3aa1-e602-4d45-a088-e7f3b57e2306   NaN      Armin Laschet   \n",
              "523fccf4-70e1-437e-bb1d-c906fd0d4648   NaN        Olaf Scholz   \n",
              "b8205fa2-9302-4ba6-82ff-00587434e0f9   NaN      Armin Laschet   \n",
              "a6c63259-e37d-4099-b55e-095b62e64bcd   NaN        Nicola Beer   \n",
              "a44eed36-cfa8-4187-b3f4-af8ccf7c5241   NaN  Christian Lindner   \n",
              "\n",
              "coder                                 Majority Decision  \n",
              "id                                                       \n",
              "0b8b3aa1-e602-4d45-a088-e7f3b57e2306      Armin Laschet  \n",
              "523fccf4-70e1-437e-bb1d-c906fd0d4648        Olaf Scholz  \n",
              "b8205fa2-9302-4ba6-82ff-00587434e0f9      Armin Laschet  \n",
              "a6c63259-e37d-4099-b55e-095b62e64bcd        Nicola Beer  \n",
              "a44eed36-cfa8-4187-b3f4-af8ccf7c5241  Christian Lindner  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7b4e6094-4cc2-4f7d-8399-949f29ec2185\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>coder</th>\n",
              "      <th>9368</th>\n",
              "      <th>11478</th>\n",
              "      <th>11766</th>\n",
              "      <th>11802</th>\n",
              "      <th>11909</th>\n",
              "      <th>11926</th>\n",
              "      <th>Majority Decision</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0b8b3aa1-e602-4d45-a088-e7f3b57e2306</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Armin Laschet</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Armin Laschet</td>\n",
              "      <td>Armin Laschet</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>523fccf4-70e1-437e-bb1d-c906fd0d4648</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Olaf Scholz</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Olaf Scholz</td>\n",
              "      <td>Olaf Scholz</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b8205fa2-9302-4ba6-82ff-00587434e0f9</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Armin Laschet</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Armin Laschet</td>\n",
              "      <td>Armin Laschet</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>a6c63259-e37d-4099-b55e-095b62e64bcd</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Nicola Beer</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Nicola Beer</td>\n",
              "      <td>Nicola Beer</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>a44eed36-cfa8-4187-b3f4-af8ccf7c5241</th>\n",
              "      <td>NaN</td>\n",
              "      <td>Christian Lindner</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Christian Lindner</td>\n",
              "      <td>Christian Lindner</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7b4e6094-4cc2-4f7d-8399-949f29ec2185')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-7b4e6094-4cc2-4f7d-8399-949f29ec2185 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-7b4e6094-4cc2-4f7d-8399-949f29ec2185');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ec2a9a7f-6ebb-48d9-8eac-46323577fcf5\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ec2a9a7f-6ebb-48d9-8eac-46323577fcf5')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ec2a9a7f-6ebb-48d9-8eac-46323577fcf5 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n5dTra3ElFhM"
      },
      "outputs": [],
      "source": [
        "# @title Import computational annotations\n",
        "# @markdown Reading the GPT annotations. Enter the correct file path below\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "annotated_file = '/content/drive/MyDrive/2024-01-04-Text-Annotation-Sample.csv' # @param {type:\"string\"}\n",
        "coding_column = 'Positioning' # @param {type:\"string\"}\n",
        "annotated_identifier = 'uuid' # @param {type:\"string\"}\n",
        "\n",
        "annotated_df = pd.read_csv(annotated_file)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "annotated_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "TU-vZS1PUSSf",
        "outputId": "88d7c23e-9595-404f-aaba-26647fe27571"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0                                               Text Post Type  \\\n",
              "0          54  Gemeinsam als #eineUnion stehen wir für eine ...      Post   \n",
              "1         200  Wir wünschen allen Schülerinnen und Schüler...      Post   \n",
              "2         530  Fränkisches Essen gibt Kraft: Nach einem Mitt...      Post   \n",
              "3         707               Kleines Zwischenfazit zum #TvTriell.      Post   \n",
              "4         899  In einem paar Tagen sind Wahlen, wichtige Wahl...     Story   \n",
              "\n",
              "   Positioning  uuid  \n",
              "0         True    54  \n",
              "1         True   200  \n",
              "2        False   530  \n",
              "3        False   707  \n",
              "4        False   899  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5ff1ac40-52a4-4ed7-a623-5dcaef9e93bb\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Text</th>\n",
              "      <th>Post Type</th>\n",
              "      <th>Positioning</th>\n",
              "      <th>uuid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>54</td>\n",
              "      <td>Gemeinsam als #eineUnion stehen wir für eine ...</td>\n",
              "      <td>Post</td>\n",
              "      <td>True</td>\n",
              "      <td>54</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>200</td>\n",
              "      <td>Wir wünschen allen Schülerinnen und Schüler...</td>\n",
              "      <td>Post</td>\n",
              "      <td>True</td>\n",
              "      <td>200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>530</td>\n",
              "      <td>Fränkisches Essen gibt Kraft: Nach einem Mitt...</td>\n",
              "      <td>Post</td>\n",
              "      <td>False</td>\n",
              "      <td>530</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>707</td>\n",
              "      <td>Kleines Zwischenfazit zum #TvTriell.</td>\n",
              "      <td>Post</td>\n",
              "      <td>False</td>\n",
              "      <td>707</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>899</td>\n",
              "      <td>In einem paar Tagen sind Wahlen, wichtige Wahl...</td>\n",
              "      <td>Story</td>\n",
              "      <td>False</td>\n",
              "      <td>899</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5ff1ac40-52a4-4ed7-a623-5dcaef9e93bb')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-5ff1ac40-52a4-4ed7-a623-5dcaef9e93bb button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-5ff1ac40-52a4-4ed7-a623-5dcaef9e93bb');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-abdf2a1f-4bfc-43f2-af45-41edb5f16c8b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-abdf2a1f-4bfc-43f2-af45-41edb5f16c8b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-abdf2a1f-4bfc-43f2-af45-41edb5f16c8b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "contingency_table = pd.merge(contingency_matrix, annotated_df[['uuid', coding_column]], left_on=identifier, right_on=annotated_identifier, how='left')\n",
        "contingency_table.rename(columns={coding_column: \"Model\"}, inplace=True)"
      ],
      "metadata": {
        "id": "xZB_oR2VUoAQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "contingency_table.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "O_RjgaGgVCY6",
        "outputId": "92c0dee4-ad0c-43a5-c24c-3ac5a8bf4065"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   uuid   7107   7379  10506  13724 Majority Decision  Model\n",
              "0  3733    NaN   True   True   True              True   True\n",
              "1  4185  False   True   True    NaN              True   True\n",
              "2   530    NaN   True  False   True              True  False\n",
              "3  2721    NaN  False  False  False             False  False\n",
              "4  3384    NaN  False  False  False             False  False"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ce102516-8808-4f4c-a545-eb231d6398fd\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uuid</th>\n",
              "      <th>7107</th>\n",
              "      <th>7379</th>\n",
              "      <th>10506</th>\n",
              "      <th>13724</th>\n",
              "      <th>Majority Decision</th>\n",
              "      <th>Model</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3733</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4185</td>\n",
              "      <td>False</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>530</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2721</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3384</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ce102516-8808-4f4c-a545-eb231d6398fd')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ce102516-8808-4f4c-a545-eb231d6398fd button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ce102516-8808-4f4c-a545-eb231d6398fd');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0a73d38b-6943-4c62-986a-ed7fee3e9049\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0a73d38b-6943-4c62-986a-ed7fee3e9049')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0a73d38b-6943-4c62-986a-ed7fee3e9049 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's suppose we're dealing with binary data. We convert all value to binary, in order to be able to compare them correctly."
      ],
      "metadata": {
        "id": "1nR_jwzzXQcR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Function to convert a column to boolean if it's not already\n",
        "def convert_to_bool(column):\n",
        "    if contingency_table[column].dtype != 'bool':\n",
        "        bool_map = {'True': True, 'False': False}\n",
        "        return contingency_table[column].map(bool_map)\n",
        "    return contingency_table[column]\n",
        "\n",
        "# Convert columns to boolean if they are not already\n",
        "contingency_table['Majority Decision']= convert_to_bool('Majority Decision')\n",
        "contingency_table['Model'] = convert_to_bool('Model')"
      ],
      "metadata": {
        "id": "tFq2hkh9XYzO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's quickly check pairwise agreement between the Model and Majority Decision using Cohen's Kappa:"
      ],
      "metadata": {
        "id": "DE8yJpDDXbMh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "kappa_scores = calculate_kappa(contingency_table[['Majority Decision', 'Model']])\n",
        "pd.DataFrame(kappa_scores).head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        },
        "id": "5k1ZQXdpVSID",
        "outputId": "eab6c82c-07be-465a-d4a2-62a5df05dae6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             Coder 1 Coder 2     Kappa  Overlap       Coding\n",
              "0  Majority Decision   Model  0.831461       50  Positioning"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6cc721fb-b428-4599-ba3d-e340d1240d96\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Coder 1</th>\n",
              "      <th>Coder 2</th>\n",
              "      <th>Kappa</th>\n",
              "      <th>Overlap</th>\n",
              "      <th>Coding</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Majority Decision</td>\n",
              "      <td>Model</td>\n",
              "      <td>0.831461</td>\n",
              "      <td>50</td>\n",
              "      <td>Positioning</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6cc721fb-b428-4599-ba3d-e340d1240d96')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6cc721fb-b428-4599-ba3d-e340d1240d96 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6cc721fb-b428-4599-ba3d-e340d1240d96');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Machine Learning Metrics\n",
        "\n",
        "Finally, let's calculate Accuracy, Precision, and F1 Score and plot a confusion matrix."
      ],
      "metadata": {
        "id": "J6GZOAQnXpyi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
        "from IPython.display import display, Markdown\n",
        "\n",
        "\n",
        "# Calculating metrics\n",
        "accuracy = accuracy_score(contingency_table['Majority Decision'], contingency_table['Model'])\n",
        "precision = precision_score(contingency_table['Majority Decision'], contingency_table['Model'], average='binary')\n",
        "recall = recall_score(contingency_table['Majority Decision'], contingency_table['Model'], average='binary')\n",
        "f1 = f1_score(contingency_table['Majority Decision'], contingency_table['Model'], average='binary')\n",
        "\n",
        "# Creating a DataFrame for the metrics\n",
        "metrics_df = pd.DataFrame({\n",
        "    'Metric': ['Accuracy', 'Precision', 'Recall', 'F1 Score'],\n",
        "    'Value': [accuracy, precision, recall, f1]\n",
        "})\n",
        "\n",
        "# Displaying the DataFrame as a table\n",
        "display(metrics_df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "id": "EgHD-Im3Xq9Z",
        "outputId": "a7713009-3b23-41fa-9e62-c538ea53e967"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "      Metric     Value\n",
              "0   Accuracy  0.940000\n",
              "1  Precision  1.000000\n",
              "2     Recall  0.769231\n",
              "3   F1 Score  0.869565"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-039310f2-6758-4b63-bd24-6ae00f05e81b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Metric</th>\n",
              "      <th>Value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Accuracy</td>\n",
              "      <td>0.940000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Precision</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Recall</td>\n",
              "      <td>0.769231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>F1 Score</td>\n",
              "      <td>0.869565</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-039310f2-6758-4b63-bd24-6ae00f05e81b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-039310f2-6758-4b63-bd24-6ae00f05e81b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-039310f2-6758-4b63-bd24-6ae00f05e81b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-b5da9eba-3318-45ab-a08a-672fc9ba512e\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b5da9eba-3318-45ab-a08a-672fc9ba512e')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-b5da9eba-3318-45ab-a08a-672fc9ba512e button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "from sklearn.metrics import confusion_matrix\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Generate the confusion matrix\n",
        "cm = confusion_matrix(contingency_table['Majority Decision'], contingency_table['Model'])\n",
        "\n",
        "# Plotting the confusion matrix\n",
        "plt.figure(figsize=(8, 6))\n",
        "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=['False', 'True'], yticklabels=['False', 'True'])\n",
        "plt.title('Confusion Matrix')\n",
        "plt.xlabel('Predicted Label')\n",
        "plt.ylabel('True Label')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 564
        },
        "id": "5Rnd7LQ0sANK",
        "outputId": "f1473ae5-ea49-4f4e-b2a0-6963d5f89823"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 800x600 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAIjCAYAAACTRapjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDnUlEQVR4nO3deVhV5f7//9dGZYMiIE5IKg6oOaGlHSNzKnNocuqkmR/BzLLUShyKylSy6FOn1MywY6Vmeiob9JNWlpqYORyHcKgcQMpKHFJBQdkorN8ffd2/djiwjc3ecj8f51rX1V5r7bXei+vqnPd53fe6t82yLEsAAAAwhp+3CwAAAEDpogEEAAAwDA0gAACAYWgAAQAADEMDCAAAYBgaQAAAAMPQAAIAABiGBhAAAMAwNIAAAACGoQEEcFF79+5Vt27dFBISIpvNpsWLF5fo9X/66SfZbDbNnTu3RK97JevcubM6d+7s7TIAlGE0gMAVID09XQ8++KAaNGiggIAABQcHq3379po+fbpOnz7t0XvHxsZqx44deu655zR//ny1bdvWo/crTXFxcbLZbAoODj7v33Hv3r2y2Wyy2Wz617/+5fb1Dxw4oEmTJik1NbUEqgWAklPe2wUAuLhly5bpn//8p+x2uwYPHqwWLVooPz9fa9eu1bhx4/T999/r3//+t0fuffr0aa1fv15PPfWURo4c6ZF7REZG6vTp06pQoYJHrn8p5cuX16lTp/Tpp5/q7rvvdjm2YMECBQQEKC8v77KufeDAAU2ePFn16tVT69ati/29L7/88rLuBwDFRQMI+LCMjAwNGDBAkZGRWrVqlWrVquU8NmLECKWlpWnZsmUeu/+RI0ckSaGhoR67h81mU0BAgMeufyl2u13t27fXf/7znyIN4MKFC3Xbbbfpo48+KpVaTp06pYoVK8rf379U7gfAXAwBAz7sxRdfVE5Ojt566y2X5u+cqKgoPfroo87PZ8+e1bPPPquGDRvKbrerXr16evLJJ+VwOFy+V69ePd1+++1au3at/vGPfyggIEANGjTQO++84zxn0qRJioyMlCSNGzdONptN9erVk/TH0Om5f/6zSZMmyWazuez76quvdOONNyo0NFRBQUFq0qSJnnzySefxC80BXLVqlTp06KBKlSopNDRUvXr10o8//nje+6WlpSkuLk6hoaEKCQnRkCFDdOrUqQv/Yf9i4MCB+vzzz5WVleXct2nTJu3du1cDBw4scv6xY8c0duxYtWzZUkFBQQoODlbPnj21bds25zmrV6/WddddJ0kaMmSIcyj53HN27txZLVq00JYtW9SxY0dVrFjR+Xf56xzA2NhYBQQEFHn+7t27q0qVKjpw4ECxnxUAJBpAwKd9+umnatCggW644YZinX///ffrmWee0bXXXqupU6eqU6dOSkpK0oABA4qcm5aWprvuuku33HKLXn75ZVWpUkVxcXH6/vvvJUl9+/bV1KlTJUn33HOP5s+fr2nTprlV//fff6/bb79dDodDiYmJevnll3XnnXfq22+/vej3VqxYoe7du+vw4cOaNGmS4uPjtW7dOrVv314//fRTkfPvvvtunTx5UklJSbr77rs1d+5cTZ48udh19u3bVzabTR9//LFz38KFC3X11Vfr2muvLXL+vn37tHjxYt1+++165ZVXNG7cOO3YsUOdOnVyNmNNmzZVYmKiJOmBBx7Q/PnzNX/+fHXs2NF5naNHj6pnz55q3bq1pk2bpi5dupy3vunTp6t69eqKjY1VQUGBJOmNN97Ql19+qRkzZigiIqLYzwoAkiQLgE/Kzs62JFm9evUq1vmpqamWJOv+++932T927FhLkrVq1SrnvsjISEuStWbNGue+w4cPW3a73RozZoxzX0ZGhiXJeumll1yuGRsba0VGRhapYeLEidaf/2tl6tSpliTryJEjF6z73D3mzJnj3Ne6dWurRo0a1tGjR537tm3bZvn5+VmDBw8ucr/77rvP5Zp9+vSxqlatesF7/vk5KlWqZFmWZd11113WzTffbFmWZRUUFFjh4eHW5MmTz/s3yMvLswoKCoo8h91utxITE537Nm3aVOTZzunUqZMlyZo1a9Z5j3Xq1Mll3/Llyy1J1pQpU6x9+/ZZQUFBVu/evS/5jABwPiSAgI86ceKEJKly5crFOv+zzz6TJMXHx7vsHzNmjCQVmSvYrFkzdejQwfm5evXqatKkifbt23fZNf/VubmDS5YsUWFhYbG+k5mZqdTUVMXFxSksLMy5Pzo6WrfccovzOf9s+PDhLp87dOigo0ePOv+GxTFw4ECtXr1aBw8e1KpVq3Tw4MHzDv9Kf8wb9PP7478+CwoKdPToUefw9tatW4t9T7vdriFDhhTr3G7duunBBx9UYmKi+vbtq4CAAL3xxhvFvhcA/BkNIOCjgoODJUknT54s1vk///yz/Pz8FBUV5bI/PDxcoaGh+vnnn132161bt8g1qlSpouPHj19mxUX1799f7du31/3336+aNWtqwIAB+uCDDy7aDJ6rs0mTJkWONW3aVL///rtyc3Nd9v/1WapUqSJJbj3LrbfeqsqVK+v999/XggULdN111xX5W55TWFioqVOnqlGjRrLb7apWrZqqV6+u7du3Kzs7u9j3vOqqq9x64eNf//qXwsLClJqaqldffVU1atQo9ncB4M9oAAEfFRwcrIiICO3cudOt7/31JYwLKVeu3Hn3W5Z12fc4Nz/tnMDAQK1Zs0YrVqzQ//zP/2j79u3q37+/brnlliLn/h1/51nOsdvt6tu3r+bNm6dPPvnkgumfJD3//POKj49Xx44d9e6772r58uX66quv1Lx582InndIffx93fPfddzp8+LAkaceOHW59FwD+jAYQ8GG333670tPTtX79+kueGxkZqcLCQu3du9dl/6FDh5SVleV8o7ckVKlSxeWN2XP+mjJKkp+fn26++Wa98sor+uGHH/Tcc89p1apV+vrrr8977XN17t69u8ixXbt2qVq1aqpUqdLfe4ALGDhwoL777judPHnyvC/OnPPhhx+qS5cueuuttzRgwAB169ZNXbt2LfI3KW4zXhy5ubkaMmSImjVrpgceeEAvvviiNm3aVGLXB2AWGkDAh40fP16VKlXS/fffr0OHDhU5np6erunTp0v6YwhTUpE3dV955RVJ0m233VZidTVs2FDZ2dnavn27c19mZqY++eQTl/OOHTtW5LvnFkT+69I059SqVUutW7fWvHnzXBqqnTt36ssvv3Q+pyd06dJFzz77rF577TWFh4df8Lxy5coVSRcXLVqk3377zWXfuUb1fM2yux5//HHt379f8+bN0yuvvKJ69eopNjb2gn9HALgYFoIGfFjDhg21cOFC9e/fX02bNnX5JZB169Zp0aJFiouLkyS1atVKsbGx+ve//62srCx16tRJ//3vfzVv3jz17t37gkuMXI4BAwbo8ccfV58+ffTII4/o1KlTSk5OVuPGjV1egkhMTNSaNWt02223KTIyUocPH9brr7+u2rVr68Ybb7zg9V966SX17NlTMTExGjp0qE6fPq0ZM2YoJCREkyZNKrHn+Cs/Pz89/fTTlzzv9ttvV2JiooYMGaIbbrhBO3bs0IIFC9SgQQOX8xo2bKjQ0FDNmjVLlStXVqVKldSuXTvVr1/frbpWrVql119/XRMnTnQuSzNnzhx17txZEyZM0IsvvujW9QCAZWCAK8CePXusYcOGWfXq1bP8/f2typUrW+3bt7dmzJhh5eXlOc87c+aMNXnyZKt+/fpWhQoVrDp16lgJCQku51jWH8vA3HbbbUXu89flRy60DIxlWdaXX35ptWjRwvL397eaNGlivfvuu0WWgVm5cqXVq1cvKyIiwvL397ciIiKse+65x9qzZ0+Re/x1qZQVK1ZY7du3twIDA63g4GDrjjvusH744QeXc87d76/LzMyZM8eSZGVkZFzwb2pZrsvAXMiFloEZM2aMVatWLSswMNBq3769tX79+vMu37JkyRKrWbNmVvny5V2es1OnTlbz5s3Pe88/X+fEiRNWZGSkde2111pnzpxxOW/06NGWn5+ftX79+os+AwD8lc2y3JglDQAAgCsecwABAAAMQwMIAABgGBpAAAAAw9AAAgAAGIYGEAAAwDA0gAAAAIahAQQAADBMmfwlkMBrRnq7BAAecnzTa94uAYCHBHixK/Fk73D6O9/77y0SQAAAAMOUyQQQAADALTazMjEaQAAAAJvN2xWUKrPaXQAAAJAAAgAAmDYEbNbTAgAAgAQQAACAOYAAAAAo00gAAQAAmAMIAACAsowEEAAAwLA5gDSAAAAADAEDAACgLCMBBAAAMGwImAQQAADAMCSAAAAAzAEEAABAWUYCCAAAwBxAAAAAlGUkgAAAAMwBBAAAMIzN5rnNDcnJyYqOjlZwcLCCg4MVExOjzz//3Hm8c+fOstlsLtvw4cPdflwSQAAAAB9Ru3ZtvfDCC2rUqJEsy9K8efPUq1cvfffdd2revLkkadiwYUpMTHR+p2LFim7fhwYQAADAR4aA77jjDpfPzz33nJKTk7VhwwZnA1ixYkWFh4f/rfv4xtMCAACUUQ6HQydOnHDZHA7HJb9XUFCg9957T7m5uYqJiXHuX7BggapVq6YWLVooISFBp06dcrsmGkAAAACbn8e2pKQkhYSEuGxJSUkXLGXHjh0KCgqS3W7X8OHD9cknn6hZs2aSpIEDB+rdd9/V119/rYSEBM2fP1+DBg1y/3Ety7Iu+4/lowKvGentEgB4yPFNr3m7BAAeEuDFiWmBnRIvfdJlyvry8SKJn91ul91uP+/5+fn52r9/v7Kzs/Xhhx/qzTffVEpKirMJ/LNVq1bp5ptvVlpamho2bFjsmpgDCAAA4Oe5haAv1uydj7+/v6KioiRJbdq00aZNmzR9+nS98cYbRc5t166dJLndADIEDAAA4MMKCwsvOGcwNTVVklSrVi23rkkCCAAA4CNvASckJKhnz56qW7euTp48qYULF2r16tVavny50tPTtXDhQt16662qWrWqtm/frtGjR6tjx46Kjo526z40gAAAAD7yW8CHDx/W4MGDlZmZqZCQEEVHR2v58uW65ZZb9Msvv2jFihWaNm2acnNzVadOHfXr109PP/202/ehAQQAAPARb7311gWP1alTRykpKSVyHxpAAAAAHxkCLi1mPS0AAABIAAEAAHxlDmBpIQEEAAAwDAkgAAAAcwABAABQlpEAAgAAGDYHkAYQAACAIWAAAACUZSSAAAAAhg0BkwACAAAYhgQQAACAOYAAAAAoy0gAAQAAmAMIAACAsowEEAAAwLA5gDSAAAAAhjWAZj0tAAAASAABAAB4CQQAAABlGgkgAAAAcwABAABQlpEAAgAAMAcQAAAAZRkJIAAAgGFzAGkAAQAAGAIGAABAWUYCCAAAjGcjAQQAAEBZRgIIAACMRwIIAACAMo0EEAAAwKwAkAQQAADANCSAAADAeKbNAaQBBAAAxjOtAWQIGAAAwDAkgAAAwHgkgAAAACjTSAABAIDxSAABAABQppEAAgAAmBUAkgACAACYhgQQAAAYjzmAAAAAKNNIAAEAgPFMSwBpAAEAgPFMawAZAgYAADAMCSAAADAeCSAAAADKNBJAAAAAswJAEkAAAADT0AACAADj2Ww2j23uSE5OVnR0tIKDgxUcHKyYmBh9/vnnzuN5eXkaMWKEqlatqqCgIPXr10+HDh1y+3lpAAEAAHxE7dq19cILL2jLli3avHmzbrrpJvXq1Uvff/+9JGn06NH69NNPtWjRIqWkpOjAgQPq27ev2/exWZZllXTx3hZ4zUhvlwDAQ45ves3bJQDwkAAvvplQfcj7Hrv2kTn9/9b3w8LC9NJLL+muu+5S9erVtXDhQt11112SpF27dqlp06Zav369rr/++mJfk5dAAACA8Ty5DIzD4ZDD4XDZZ7fbZbfbL/q9goICLVq0SLm5uYqJidGWLVt05swZde3a1XnO1Vdfrbp167rdADIEDAAA4EFJSUkKCQlx2ZKSki54/o4dOxQUFCS73a7hw4frk08+UbNmzXTw4EH5+/srNDTU5fyaNWvq4MGDbtVEAggAAODBZWASEhIUHx/vsu9i6V+TJk2Umpqq7Oxsffjhh4qNjVVKSkqJ1kQDCAAA4EHFGe79M39/f0VFRUmS2rRpo02bNmn69Onq37+/8vPzlZWV5ZICHjp0SOHh4W7VxBAwAAAwnq8sA3M+hYWFcjgcatOmjSpUqKCVK1c6j+3evVv79+9XTEyMW9ckAQQAAPARCQkJ6tmzp+rWrauTJ09q4cKFWr16tZYvX66QkBANHTpU8fHxCgsLU3BwsEaNGqWYmBi3XgCRaAABAAA8+hawOw4fPqzBgwcrMzNTISEhio6O1vLly3XLLbdIkqZOnSo/Pz/169dPDodD3bt31+uvv+72fVgHEMAVhXUAgbLLm+sAhg/70GPXPjj7Lo9d+3KRAAIAAOP5SgJYWmgAAQCA8UxrAHkLGAAAwDAkgAAAAGYFgCSAAAAApiEBBAAAxmMOIAAAAMo0n2oA8/PztXv3bp09e9bbpQAAAIP48k/BeYJPNICnTp3S0KFDVbFiRTVv3lz79++XJI0aNUovvPCCl6sDAAAoW3yiAUxISNC2bdu0evVqBQQEOPd37dpV77//vhcrAwAAJjAtAfSJl0AWL16s999/X9dff73LH6p58+ZKT0/3YmUAAMAIvtmneYxPJIBHjhxRjRo1iuzPzc312c4ZAADgSuUTDWDbtm21bNky5+dzTd+bb76pmJgYb5UFAAAMwRCwFzz//PPq2bOnfvjhB509e1bTp0/XDz/8oHXr1iklJcXb5QEAAJQpPpEA3njjjUpNTdXZs2fVsmVLffnll6pRo4bWr1+vNm3aeLs8AABQxpEAeknDhg01e/Zsb5cBAABQ5vlEA7h161ZVqFBBLVu2lCQtWbJEc+bMUbNmzTRp0iT5+/t7uUJ427B/3qhhd3VQZESYJOnHfQf1/L8/15ff/qC6tcK0+7PE837v3nFv6eMV35VmqQBKyHsLF2jenLf0++9H1LjJ1XriyQlqGR3t7bJQRvlqUucpPjEE/OCDD2rPnj2SpH379ql///6qWLGiFi1apPHjx3u5OviC3w5lacKMJbrh3hfV/t6XtPq/e7Ro6gNq2iBcvx46rnpdE1y2xOSlOpmbp+Xffu/t0gFchi8+/0z/ejFJDz48Qu8t+kRNmlythx4cqqNHj3q7NKBM8IkGcM+ePWrdurUkadGiRerUqZMWLlyouXPn6qOPPvJucfAJn63ZqeVrf1D6/iNK239Yk2Z+qpxTDv0jur4KCy0dOnrSZbuzSyt99NVW5Z7O93bpAC7D/Hlz1Peuu9W7Tz81jIrS0xMnKyAgQIs/5n8T4BmmzQH0iQbQsiwVFhZKklasWKFbb71VklSnTh39/vvv3iwNPsjPz6Z/dm+jSoH+2rg9o8jxa5rWUeur62je4vVeqA7A33UmP18//vC9ro+5wbnPz89P119/g7ZvY0oHPMTmwc0H+cQcwLZt22rKlCnq2rWrUlJSlJycLEnKyMhQzZo1L/pdh8Mhh8Phss8qLJDNr5zH6oV3NI+K0Op5YxTgX145px3qP2a2du07WOS82N4x+nFfpjZsK9ocAvB9x7OOq6CgQFWrVnXZX7VqVWVk7PNSVUDZ4hMJ4LRp07R161aNHDlSTz31lKKioiRJH374oW644YaLfjcpKUkhISEu29lDW0qjbJSyPT8dUrsBSeo4+F+avWitZif+j65uEO5yToC9gvr3bEv6BwBwi2lDwD6RAEZHR2vHjh1F9r/00ksqV+7iSV5CQoLi4+Nd9tXo8HiJ1gffcOZsgfb98seUgO9+/EVtmtfViHs6a9Rz7znP6dO1tSoG+GvB0v96q0wAf1OV0CoqV65ckRc+jh49qmrVqnmpKqBs8YkE8EICAgJUoUKFi55jt9sVHBzssjH8awY/m012f9f/DxPX+wYtS9mh34/neKkqAH9XBX9/NW3WXBs3/P9JfmFhoTZuXK/oVtd4sTKUZSSApaRKlSrF/qMcO3bMw9XA1yWOulPLv/1ev2QeV+VKAerfs606tm2kOx5+3XlOgzrVdOO1DdV7VLIXKwVQEv4ndogmPPm4mjdvoRYto/Xu/Hk6ffq0evfp6+3SgDLBaw3gtGnTvHVrXIGqhwXprWcHK7xasLJz8rRz72+64+HXtWrjLuc5sb1i9NuhLK1Yv+siVwJwJejR81YdP3ZMr7/2qn7//YiaXN1Ur7/xpqoyBAwP8dGgzmNslmVZ3i6ipAVeM9LbJQDwkOObXvN2CQA8JMCLbyZEjf3cY9dO+1dPj137cvnESyB/lpeXp/x818V7g4ODvVQNAAAwga/O1fMUn3gJJDc3VyNHjlSNGjVUqVIlValSxWUDAADwJJvNc5sv8okGcPz48Vq1apWSk5Nlt9v15ptvavLkyYqIiNA777zj7fIAAADKFJ8YAv7000/1zjvvqHPnzhoyZIg6dOigqKgoRUZGasGCBbr33nu9XSIAACjDGAL2gmPHjqlBgwaS/pjvd27ZlxtvvFFr1qzxZmkAAABljk80gA0aNFBGxh+/23r11Vfrgw8+kPRHMhgaGurFygAAgAmYA1iK9u3bp8LCQg0ZMkTbtm2TJD3xxBOaOXOmAgICNHr0aI0bN86bJQIAAJQ5Xp0D2KhRI2VmZmr06NGSpP79++vVV1/Vrl27tGXLFkVFRSk6OtqbJQIAAAP4+floVOchXk0A/7oG9Weffabc3FxFRkaqb9++NH8AAAAe4BNvAQMAAHiTr87V8xSvNoA2m63Ia9emvYYNAAC8z7T+w6sNoGVZiouLk91ul/THz8ANHz5clSpVcjnv448/9kZ5AAAAZZJXG8DY2FiXz4MGDfJSJQAAwGSGBYDebQDnzJnjzdsDAAAYiZdAAACA8UybA+gTvwQCAACA0kMCCAAAjEcCCAAAgDKNBBAAABjPsACQBhAAAIAhYAAAAJRpJIAAAMB4hgWAJIAAAACmIQEEAADGYw4gAAAAyjQaQAAAYDybzXObO5KSknTdddepcuXKqlGjhnr37q3du3e7nNO5c2fZbDaXbfjw4W7dhwYQAADAR6SkpGjEiBHasGGDvvrqK505c0bdunVTbm6uy3nDhg1TZmamc3vxxRfdug9zAAEAgPF8ZQ7gF1984fJ57ty5qlGjhrZs2aKOHTs691esWFHh4eGXfR8SQAAAAA9yOBw6ceKEy+ZwOIr13ezsbElSWFiYy/4FCxaoWrVqatGihRISEnTq1Cm3aqIBBAAAxvPkHMCkpCSFhIS4bElJSZesqbCwUI899pjat2+vFi1aOPcPHDhQ7777rr7++mslJCRo/vz5GjRokFvPyxAwAAAwnieHgBMSEhQfH++yz263X/J7I0aM0M6dO7V27VqX/Q888IDzn1u2bKlatWrp5ptvVnp6uho2bFismmgAAQAAPMhutxer4fuzkSNHaunSpVqzZo1q16590XPbtWsnSUpLS6MBBAAAKC4feQdElmVp1KhR+uSTT7R69WrVr1//kt9JTU2VJNWqVavY96EBBAAA8BEjRozQwoULtWTJElWuXFkHDx6UJIWEhCgwMFDp6elauHChbr31VlWtWlXbt2/X6NGj1bFjR0VHRxf7PjSAAADAeL6yDExycrKkPxZ7/rM5c+YoLi5O/v7+WrFihaZNm6bc3FzVqVNH/fr109NPP+3WfWgAAQAAfIRlWRc9XqdOHaWkpPzt+9AAAgAA4/lIAFhqWAcQAADAMCSAAADAeL4yB7C00AACAADjGdb/MQQMAABgGhJAAABgPNOGgEkAAQAADEMCCAAAjEcCCAAAgDKNBBAAABjPsACQBBAAAMA0JIAAAMB4ps0BpAEEAADGM6z/YwgYAADANCSAAADAeKYNAZMAAgAAGIYEEAAAGM+wAJAEEAAAwDQkgAAAwHh+hkWAJIAAAACGIQEEAADGMywApAEEAABgGRgAAACUaSSAAADAeH5mBYAkgAAAAKYhAQQAAMZjDiAAAADKNBJAAABgPMMCQBJAAAAA05AAAgAA49lkVgRIAwgAAIzHMjAAAAAo00gAAQCA8VgGBgAAAGUaCSAAADCeYQEgCSAAAIBpSAABAIDx/AyLAEkAAQAADEMCCAAAjGdYAEgDCAAAYNoyMMVqALdv317sC0ZHR192MQAAAPC8YjWArVu3ls1mk2VZ5z1+7pjNZlNBQUGJFggAAOBphgWAxWsAMzIyPF0HAAAASkmxGsDIyEhP1wEAAOA1LANTDPPnz1f79u0VERGhn3/+WZI0bdo0LVmypESLAwAAQMlzuwFMTk5WfHy8br31VmVlZTnn/IWGhmratGklXR8AAIDH2Ty4+SK3G8AZM2Zo9uzZeuqpp1SuXDnn/rZt22rHjh0lWhwAAABKntvrAGZkZOiaa64pst9utys3N7dEigIAAChNpq0D6HYCWL9+faWmphbZ/8UXX6hp06YlURMAAECp8rN5bvNFbieA8fHxGjFihPLy8mRZlv773//qP//5j5KSkvTmm296okYAAACUILcbwPvvv1+BgYF6+umnderUKQ0cOFARERGaPn26BgwY4IkaAQAAPIoh4GK49957tXfvXuXk5OjgwYP69ddfNXTo0JKuDQAAwChJSUm67rrrVLlyZdWoUUO9e/fW7t27Xc7Jy8vTiBEjVLVqVQUFBalfv346dOiQW/e5rAZQkg4fPqwtW7Zo9+7dOnLkyOVeBgAAwOtsNs9t7khJSdGIESO0YcMGffXVVzpz5oy6devm8qLt6NGj9emnn2rRokVKSUnRgQMH1LdvX7fu4/YQ8MmTJ/Xwww/rP//5jwoLCyVJ5cqVU//+/TVz5kyFhIS4e0kAAADoj5dq/2zu3LmqUaOGtmzZoo4dOyo7O1tvvfWWFi5cqJtuukmSNGfOHDVt2lQbNmzQ9ddfX6z7uJ0A3n///dq4caOWLVumrKwsZWVlaenSpdq8ebMefPBBdy8HAADgdTabzWObw+HQiRMnXDaHw1GsurKzsyVJYWFhkqQtW7bozJkz6tq1q/Ocq6++WnXr1tX69euL/bxuN4BLly7V22+/re7duys4OFjBwcHq3r27Zs+erU8//dTdywEAAJRpSUlJCgkJcdmSkpIu+b3CwkI99thjat++vVq0aCFJOnjwoPz9/RUaGupybs2aNXXw4MFi1+T2EHDVqlXPO8wbEhKiKlWquHs5AAAAr/Pken0JCQmKj4932We32y/5vREjRmjnzp1au3ZtidfkdgL49NNPKz4+3qXLPHjwoMaNG6cJEyaUaHEAAAClwZNDwHa73Tlqem67VAM4cuRILV26VF9//bVq167t3B8eHq78/HxlZWW5nH/o0CGFh4cX+3mLlQBec801Luvj7N27V3Xr1lXdunUlSfv375fdbteRI0eYBwgAAHCZLMvSqFGj9Mknn2j16tWqX7++y/E2bdqoQoUKWrlypfr16ydJ2r17t/bv36+YmJhi36dYDWDv3r2LXzkAAMAVxleWgR4xYoQWLlyoJUuWqHLlys4R15CQEAUGBiokJERDhw5VfHy8wsLCFBwcrFGjRikmJqbYbwBLxWwAJ06ceHlPAQAAgGJLTk6WJHXu3Nll/5w5cxQXFydJmjp1qvz8/NSvXz85HA51795dr7/+ulv3cfslEAAAgLLGz0d+Cs6yrEueExAQoJkzZ2rmzJmXfR+3G8CCggJNnTpVH3zwgfbv36/8/HyX48eOHbvsYgAAAOB5br8FPHnyZL3yyivq37+/srOzFR8fr759+8rPz0+TJk3yQIkAAACe5Ss/BVda3G4AFyxYoNmzZ2vMmDEqX7687rnnHr355pt65plntGHDBk/UCAAAgBLkdgN48OBBtWzZUpIUFBTk/ImS22+/XcuWLSvZ6gAAAEqBJ9cB9EVuN4C1a9dWZmamJKlhw4b68ssvJUmbNm0q1qrWAAAA8C63G8A+ffpo5cqVkqRRo0ZpwoQJatSokQYPHqz77ruvxAsEAADwNNPmALr9FvALL7zg/Of+/fsrMjJS69atU6NGjXTHHXeUaHEAAAClwVeWgSktbieAf3X99dcrPj5e7dq10/PPP18SNQEAAMCD/nYDeE5mZqYmTJhQUpcDAAAoNaYNAZdYAwgAAIArAz8FBwAAjOery7V4CgkgAACAYYqdAMbHx1/0+JEjR/52MSUlI2Wqt0sA4CEb0vm9caCs6twkzGv3Ni0RK3YD+N13313ynI4dO/6tYgAAAOB5xW4Av/76a0/WAQAA4DWmzQHkJRAAAGA8P7P6P+OGvAEAAIxHAggAAIxHAggAAIAyjQQQAAAYz7SXQC4rAfzmm280aNAgxcTE6LfffpMkzZ8/X2vXri3R4gAAAFDy3G4AP/roI3Xv3l2BgYH67rvv5HA4JEnZ2dl6/vnnS7xAAAAAT/OzeW7zRW43gFOmTNGsWbM0e/ZsVahQwbm/ffv22rp1a4kWBwAAgJLn9hzA3bt3n/cXP0JCQpSVlVUSNQEAAJQqw6YAup8AhoeHKy0trcj+tWvXqkGDBiVSFAAAQGnys9k8tvkitxvAYcOG6dFHH9XGjRtls9l04MABLViwQGPHjtVDDz3kiRoBAABQgtweAn7iiSdUWFiom2++WadOnVLHjh1lt9s1duxYjRo1yhM1AgAAeJRpCyO73QDabDY99dRTGjdunNLS0pSTk6NmzZopKCjIE/UBAACghF32QtD+/v5q1qxZSdYCAADgFT46Vc9j3G4Au3TpctHVsletWvW3CgIAAIBnud0Atm7d2uXzmTNnlJqaqp07dyo2Nrak6gIAACg1vvq2rqe43QBOnTr1vPsnTZqknJycv10QAAAAPKvEXnoZNGiQ3n777ZK6HAAAQKmx2Ty3+aLLfgnkr9avX6+AgICSuhwAAECp8dXf7PUUtxvAvn37uny2LEuZmZnavHmzJkyYUGKFAQAAwDPcbgBDQkJcPvv5+alJkyZKTExUt27dSqwwAACA0sJLIBdRUFCgIUOGqGXLlqpSpYqnagIAAIAHufUSSLly5dStWzdlZWV5qBwAAIDSZ9pLIG6/BdyiRQvt27fPE7UAAACgFLjdAE6ZMkVjx47V0qVLlZmZqRMnTrhsAAAAVxo/m+c2X1TsOYCJiYkaM2aMbr31VknSnXfe6fKTcJZlyWazqaCgoOSrBAAAQIkpdgM4efJkDR8+XF9//bUn6wEAACh1NvloVOchxW4ALcuSJHXq1MljxQAAAHiDrw7VeopbcwBtvvoqCwAAAIrNrXUAGzdufMkm8NixY3+rIAAAgNJmWgLoVgM4efLkIr8EAgAAgCuLWw3ggAEDVKNGDU/VAgAA4BWmTXMr9hxA0/4wAAAAZZXbbwEDAACUNcwBvIDCwkJP1gEAAIBS4tYcQAAAgLLItJluNIAAAMB4foZ1gG4tBA0AAADPWrNmje644w5FRETIZrNp8eLFLsfj4uJks9lcth49erh1DxJAAABgPF96CSQ3N1etWrXSfffdp759+573nB49emjOnDnOz3a73a170AACAAB4kMPhkMPhcNlnt9sv2LT17NlTPXv2vOg17Xa7wsPDL7smhoABAIDxbDbPbUlJSQoJCXHZkpKS/la9q1evVo0aNdSkSRM99NBDOnr0qFvfJwEEAADwoISEBMXHx7vsc3fI9s969Oihvn37qn79+kpPT9eTTz6pnj17av369SpXrlyxrkEDCAAAjOcnz00CvNhw7+UYMGCA859btmyp6OhoNWzYUKtXr9bNN99crGswBAwAAHAFa9CggapVq6a0tLRif4cEEAAAGO9KXgbw119/1dGjR1WrVq1if4cGEAAAGM+XloHJyclxSfMyMjKUmpqqsLAwhYWFafLkyerXr5/Cw8OVnp6u8ePHKyoqSt27dy/2PWgAAQAAfMjmzZvVpUsX5+dzL5DExsYqOTlZ27dv17x585SVlaWIiAh169ZNzz77rFvzDGkAAQCA8Xzpp+A6d+4sy7IueHz58uV/+x68BAIAAGAYEkAAAGA8HwoASwUJIAAAgGFIAAEAgPF8aQ5gaSABBAAAMAwJIAAAMJ5hASANIAAAgGlDoqY9LwAAgPFIAAEAgPFsho0BkwACAAAYhgQQAAAYz6z8jwQQAADAOCSAAADAeCwEDQAAgDKNBBAAABjPrPyPBhAAAMC4XwJhCBgAAMAwJIAAAMB4LAQNAACAMo0EEAAAGM+0RMy05wUAADAeCSAAADAecwABAABQppEAAgAA45mV/5EAAgAAGIcEEAAAGM+0OYA0gAAAwHimDYma9rwAAADGIwEEAADGM20ImAQQAADAMCSAAADAeGblfySAAAAAxiEBBAAAxjNsCiAJIAAAgGlIAAEAgPH8DJsFSAMIAACMxxAwAAAAyjQSQAAAYDybYUPAJIAAAACGIQEEAADGYw4gAAAAyjQSQAAAYDzTloEhAQQAADAMCSAAADCeaXMAaQABAIDxTGsAGQIGAAAwDAkgAAAwHgtBAwAAoEwjAQQAAMbzMysAJAEEAAAwDQkgAAAwHnMAAQAA4DVr1qzRHXfcoYiICNlsNi1evNjluGVZeuaZZ1SrVi0FBgaqa9eu2rt3r1v38IkG8JtvvtGgQYMUExOj3377TZI0f/58rV271suVAQAAE9hsntvclZubq1atWmnmzJnnPf7iiy/q1Vdf1axZs7Rx40ZVqlRJ3bt3V15eXrHv4fUG8KOPPlL37t0VGBio7777Tg6HQ5KUnZ2t559/3svVAQAAE9g8+B939ezZU1OmTFGfPn2KHLMsS9OmTdPTTz+tXr16KTo6Wu+8844OHDhQJCm8GK83gFOmTNGsWbM0e/ZsVahQwbm/ffv22rp1qxcrAwAA+PscDodOnDjhsp0LvNyVkZGhgwcPqmvXrs59ISEhateundavX1/s63i9Ady9e7c6duxYZH9ISIiysrJKvyAAAGAcP5vntqSkJIWEhLhsSUlJl1XnwYMHJUk1a9Z02V+zZk3nseLw+lvA4eHhSktLU7169Vz2r127Vg0aNPBOUQAAACUkISFB8fHxLvvsdruXqvmD1xPAYcOG6dFHH9XGjRtls9l04MABLViwQGPHjtVDDz3k7fIAAIABPDkH0G63Kzg42GW73AYwPDxcknTo0CGX/YcOHXIeKw6vJ4BPPPGECgsLdfPNN+vUqVPq2LGj7Ha7xo4dq1GjRnm7PAAAAJ9Rv359hYeHa+XKlWrdurUk6cSJE9q4caNbwZnXG0CbzaannnpK48aNU1pamnJyctSsWTMFBQV5uzT4sMUfvqclH72vg5kHJEn1GkQpduhwXd++g5crA+CuPTu/05efLND+9N3KPva7HnryBbW+vpPzuGVZ+nThbH3z5f/pdO5JNWwarYEPjVfNiDperBplzeUs1+IpOTk5SktLc37OyMhQamqqwsLCVLduXT322GOaMmWKGjVqpPr162vChAmKiIhQ7969i30PrzeA5/j7+6tZs2beLgNXiOo1wvXgyNGqXSdSlmXpi2VL9NTYUXrz3Q9Vv2GUt8sD4IZ8R55q12+k9l1v16ykhCLHl3/8rlYtXaS4RyeoWs0I/d+Cf+vViY9p0syFquDv3XlUgCds3rxZXbp0cX4+N38wNjZWc+fO1fjx45Wbm6sHHnhAWVlZuvHGG/XFF18oICCg2PewWZZllXjlbujSpYtsF2m7V61a5fY1D54483dKwhXq9ptv0EOPjNFtvfp5uxR40K7Mk94uAR704J0xLgmgZVkaH3eHbul9j7r1uVeSdDo3R2MH36a4R5/WdR1v8Wa5KGGdm4R57d7f7j3usWu3b1TFY9e+XF5PAM+NX59z5swZpaamaufOnYqNjfVOUbiiFBQUaPXK5co7fVrNW7b2djkAStDvhw7oxPGjatrqOue+wEpBqt+4mfbt3kkDiBLj50tjwKXA6w3g1KlTz7t/0qRJysnJueT3HQ5HkcUUHQ4/r79eDc9LT9ujEffdq/z8fAUGVtSUl6arXoOG3i4LQAk6cfyoJCk41DUZCg4NU/b/OwbAfV5fBuZCBg0apLfffvuS551vccUZr/xvKVQIb6sbWV9vLvhIyXMWqle/u/X8pKf00750b5cFALgC2Ty4+SKfbQDXr19frMmMCQkJys7OdtlGxT9eChXC2ypUqKDadeqqSdPmemDkaEU1aqIP33vX22UBKEHBVapKkk5kHXPZfyLrmEL+3zEA7vP6EHDfvn1dPluWpczMTG3evFkTJky45PftdnuR4d5TvARipEKrUGfy871dBoASVK1mhIKrVNWubZtVp0FjSdLpU7nK2PODOvXse4lvA27w1ajOQ7zeAIaEhLh89vPzU5MmTZSYmKhu3bp5qSr4un+/NlXtbuigGuG1dOpUrlZ+sUypWzbppRlveLs0AG7KO31KRzJ/dX7+/dAB/bJvjypVDlZY9XDdfGd/ffbBXNWIqKNqNWtpyYLZCg2rptbXF/0deQDF49UGsKCgQEOGDFHLli1VpYrvvSIN33X8+DE9P+lJHf39iCoFVVbDqMZ6acYbuq7dDd4uDYCbfk7bpVeeGuH8vOitVyVJMTfdqrjHJqh730HKzzutd2e+oFO5OYpqFq1HJk1lDUCUKJthEaDX1wEMCAjQjz/+qPr165fYNVkHECi7WAcQKLu8uQ7gxvRsj127XcOQS59Uyrz+EkiLFi20b98+b5cBAAAMZrN5bvNFXm8Ap0yZorFjx2rp0qXKzMzUiRMnXDYAAABPM20ZGK/NAUxMTNSYMWN06623SpLuvPNOl5+EsyxLNptNBQUF3ioRAACgTPLaHMBy5copMzNTP/7440XP69Spk9vXZg4gUHYxBxAou7w5B3BThufmAF5X3/fmAHotATzXd15OgwcAAIDL59VlYGy+OjMSAAAYxbRlYLzaADZu3PiSTeCxY8cuehwAAADu8WoDOHny5CK/BAIAAFDaTBuU9GoDOGDAANWoUcObJQAAABjHaw0g8/8AAICvMK0r8fpbwAAAAF5nWAfotQawsLDQW7cGAAAwmlfnAAIAAPgC05aB8fpvAQMAAKB0kQACAADjmfZuKgkgAACAYUgAAQCA8QwLAEkAAQAATEMCCAAAYFgESAMIAACMxzIwAAAAKNNIAAEAgPFYBgYAAABlGgkgAAAwnmEBIAkgAACAaUgAAQAADIsASQABAAAMQwIIAACMxzqAAAAAKNNIAAEAgPFMWweQBhAAABjPsP6PIWAAAADTkAACAAAYFgGSAAIAABiGBBAAABiPZWAAAABQppEAAgAA45m2DAwJIAAAgGFIAAEAgPEMCwBpAAEAAEzrABkCBgAAMAwJIAAAMB7LwAAAAKBMowEEAADGs9k8t7lj0qRJstlsLtvVV19d4s/LEDAAAIAPad68uVasWOH8XL58ybdrNIAAAMB4vjQDsHz58goPD/foPRgCBgAA8CCHw6ETJ064bA6H44Ln7927VxEREWrQoIHuvfde7d+/v8RrogEEAACweW5LSkpSSEiIy5aUlHTeMtq1a6e5c+fqiy++UHJysjIyMtShQwedPHmyZB/XsiyrRK/oAw6eOOPtEgB4yK7Mkv0vQQC+o3OTMK/de9+RPI9d+6pgW5HEz263y263X/K7WVlZioyM1CuvvKKhQ4eWWE3MAQQAAPCg4jZ75xMaGqrGjRsrLS2tRGtiCBgAABjPV5aB+aucnBylp6erVq1aJfOg/w8NIAAAgI8YO3asUlJS9NNPP2ndunXq06ePypUrp3vuuadE78MQMAAAMJ6vLAPz66+/6p577tHRo0dVvXp13XjjjdqwYYOqV69eovehAQQAAPAR7733XqnchwYQAADAVyLAUsIcQAAAAMOQAAIAAOPZDIsAaQABAIDx/u5yLVcahoABAAAMQwIIAACMZ1gASAIIAABgGhJAAABgPOYAAgAAoEwjAQQAADBsFiAJIAAAgGFIAAEAgPFMmwNIAwgAAIxnWP/HEDAAAIBpSAABAIDxTBsCJgEEAAAwDAkgAAAwns2wWYAkgAAAAIYhAQQAADArACQBBAAAMA0JIAAAMJ5hASANIAAAAMvAAAAAoEwjAQQAAMZjGRgAAACUaSSAAAAAZgWAJIAAAACmIQEEAADGMywAJAEEAAAwDQkgAAAwnmnrANIAAgAA47EMDAAAAMo0EkAAAGA804aASQABAAAMQwMIAABgGBpAAAAAwzAHEAAAGI85gAAAACjTSAABAIDxTFsHkAYQAAAYjyFgAAAAlGkkgAAAwHiGBYAkgAAAAKYhAQQAADAsAiQBBAAAMAwJIAAAMJ5py8CQAAIAABiGBBAAABiPdQABAABQppEAAgAA4xkWANIAAgAAmNYBMgQMAABgGBpAAABgPJsH/3M5Zs6cqXr16ikgIEDt2rXTf//73xJ9XhpAAAAAH/L+++8rPj5eEydO1NatW9WqVSt1795dhw8fLrF72CzLskrsaj7i4Ikz3i4BgIfsyjzp7RIAeEjnJmFeu3feWc9dO8DNNy7atWun6667Tq+99pokqbCwUHXq1NGoUaP0xBNPlEhNJIAAAAAe5HA4dOLECZfN4XCc99z8/Hxt2bJFXbt2de7z8/NT165dtX79+hKrqUy+BRweXMHbJaCUOBwOJSUlKSEhQXa73dvloBSEB3svIUDp4t9vlCZ3Uzp3TJqSpMmTJ7vsmzhxoiZNmlTk3N9//10FBQWqWbOmy/6aNWtq165dJVZTmRwChjlOnDihkJAQZWdnKzg42NvlAChB/PuNssLhcBRJ/Ox2+3n/j82BAwd01VVXad26dYqJiXHuHz9+vFJSUrRx48YSqalMJoAAAAC+4kLN3vlUq1ZN5cqV06FDh1z2Hzp0SOHh4SVWE3MAAQAAfIS/v7/atGmjlStXOvcVFhZq5cqVLong30UCCAAA4EPi4+MVGxurtm3b6h//+IemTZum3NxcDRkypMTuQQOIK5rdbtfEiROZIA6UQfz7DVP1799fR44c0TPPPKODBw+qdevW+uKLL4q8GPJ38BIIAACAYZgDCAAAYBgaQAAAAMPQAAIAABiGBhBXrLlz5yo0NNTbZQAAcMWhAYTXxcXFyWazFdnS0tK8XRqAEnK+f8f/vJ3vJ7EAeA7LwMAn9OjRQ3PmzHHZV716dS9VA6CkZWZmOv/5/fff1zPPPKPdu3c79wUFBTn/2bIsFRQUqHx5/icK8BQSQPgEu92u8PBwl2369Olq2bKlKlWqpDp16ujhhx9WTk7OBa+xbds2denSRZUrV1ZwcLDatGmjzZs3O4+vXbtWHTp0UGBgoOrUqaNHHnlEubm5pfF4gPH+/O92SEiIbDab8/OuXbtUuXJlff7552rTpo3sdrvWrl2ruLg49e7d2+U6jz32mDp37uz8XFhYqKSkJNWvX1+BgYFq1aqVPvzww9J9OOAKRAMIn+Xn56dXX31V33//vebNm6dVq1Zp/PjxFzz/3nvvVe3atbVp0yZt2bJFTzzxhCpUqCBJSk9PV48ePdSvXz9t375d77//vtauXauRI0eW1uMAuIQnnnhCL7zwgn788UdFR0cX6ztJSUl65513NGvWLH3//fcaPXq0Bg0apJSUFA9XC1zZyNfhE5YuXeoyBNSzZ08tWrTI+blevXqaMmWKhg8frtdff/2819i/f7/GjRunq6++WpLUqFEj57GkpCTde++9euyxx5zHXn31VXXq1EnJyckKCAjwwFMBcEdiYqJuueWWYp/vcDj0/PPPa8WKFc7fSG3QoIHWrl2rN954Q506dfJUqcAVjwYQPqFLly5KTk52fq5UqZJWrFihpKQk7dq1SydOnNDZs2eVl5enU6dOqWLFikWuER8fr/vvv1/z589X165d9c9//lMNGzaU9Mfw8Pbt27VgwQLn+ZZlqbCwUBkZGWratKnnHxLARbVt29at89PS0nTq1KkiTWN+fr6uueaakiwNKHNoAOETKlWqpKioKOfnn376SbfffrseeughPffccwoLC9PatWs1dOhQ5efnn7cBnDRpkgYOHKhly5bp888/18SJE/Xee++pT58+ysnJ0YMPPqhHHnmkyPfq1q3r0WcDUDyVKlVy+ezn56e//lrpmTNnnP98bk7wsmXLdNVVV7mcx+8HAxdHAwiftGXLFhUWFurll1+Wn98fU1U/+OCDS36vcePGaty4sUaPHq177rlHc+bMUZ8+fXTttdfqhx9+cGkyAfi26tWra+fOnS77UlNTnXN7mzVrJrvdrv379zPcC7iJl0Dgk6KionTmzBnNmDFD+/bt0/z58zVr1qwLnn/69GmNHDlSq1ev1s8//6xvv/1WmzZtcg7tPv7441q3bp1Gjhyp1NRU7d27V0uWLOElEMCH3XTTTdq8ebPeeecd7d27VxMnTnRpCCtXrqyxY8dq9OjRmjdvntLT07V161bNmDFD8+bN82LlgO+jAYRPatWqlV555RX97//+r1q0aKEFCxYoKSnpgueXK1dOR48e1eDBg9W4cWPdfffd6tmzpyZPnixJio6OVkpKivbs2aMOHTrommuu0TPPPKOIiIjSeiQAburevbsmTJig8ePH67rrrtPJkyc1ePBgl3OeffZZTZgwQUlJSWratKl69OihZcuWqX79+l6qGrgy2Ky/TrAAAABAmUYCCAAAYBgaQAAAAMPQAAIAABiGBhAAAMAwNIAAAACGoQEEAAAwDA0gAACAYWgAAQAADEMDCKDExMXFqXfv3s7PnTt31mOPPVbqdaxevVo2m01ZWVkeu8dfn/VylEadAHA+NIBAGRcXFyebzSabzSZ/f39FRUUpMTFRZ8+e9fi9P/74Yz377LPFOre0m6F69epp2rRppXIvAPA15b1dAADP69Gjh+bMmSOHw6HPPvtMI0aMUIUKFZSQkFDk3Pz8fPn7+5fIfcPCwkrkOgCAkkUCCBjAbrcrPDxckZGReuihh9S1a1f93//9n6T/fyjzueeeU0REhJo0aSJJ+uWXX3T33XcrNDRUYWFh6tWrl3766SfnNQsKChQfH6/Q0FBVrVpV48eP119/WvyvQ8AOh0OPP/646tSpI7vdrqioKL311lv66aef1KVLF0lSlSpVZLPZFBcXJ0kqLCxUUlKS6tevr8DAQLVq1Uoffvihy30+++wzNW7cWIGBgerSpYtLnZejoKBAQ4cOdd6zSZMmmj59+nnPnTx5sqpXr67g4GANHz5c+fn5zmPFqR0AvIEEEDBQYGCgjh496vy8cuVKBQcH66uvvpIknTlzRt27d1dMTIy++eYblS9fXlOmTFGPHj20fft2+fv76+WXX9bcuXP19ttvq2nTpnr55Zf1ySef6KabbrrgfQcPHqz169fr1VdfVatWrZSRkaHff/9dderU0UcffaR+/fpp9+7dCg4OVmBgoCQpKSlJ7777rmbNmqVGjRppzZo1GjRokKpXr65OnTrpl19+Ud++fTVixAg98MAD2rx5s8aMGfO3/j6FhYWqXbu2Fi1apKpVq2rdunV64IEHVKtWLd19990uf7eAgACtXr1aP/30k4YMGaKqVavqueeeK1btAOA1FoAyLTY21urVq5dlWZZVWFhoffXVV5bdbrfGjh3rPF6zZk3L4XA4vzN//nyrSZMmVmFhoXOfw+GwAgMDreXLl1uWZVm1atWyXnzxRefxM2fOWLVr13bey7Isq1OnTtajjz5qWZZl7d6925JkffXVV+et8+uvv7YkWcePH3fuy8vLsypWrGitW7fO5dyhQ4da99xzj2VZlpWQkGA1a9bM5fjjjz9e5Fp/FRkZaU2dOvWCx/9qxIgRVr9+/ZyfY2NjrbCwMCs3N9e5Lzk52QoKCrIKCgqKVfv5nhkASgMJIGCApUuXKigoSGfOnFFhYaEGDhyoSZMmOY+3bNnSZd7ftm3blJaWpsqVK7tcJy8vT+np6crOzlZmZqbatWvnPFa+fHm1bdu2yDDwOampqSpXrpxbyVdaWppOnTqlW265xWV/fn6+rrnmGknSjz/+6FKHJMXExBT7Hhcyc+ZMvf3229q/f79Onz6t/Px8tW7d2uWcVq1aqWLFii73zcnJ0S+//KKcnJxL1g4A3kIDCBigS5cuSk5Olr+/vyIiIlS+vOu/+pUqVXL5nJOTozZt2mjBggVFrlW9evXLquHckK47cnJyJEnLli3TVVdd5XLMbrdfVh3F8d5772ns2LF6+eWXFRMTo8qVK+ull17Sxo0bi30Nb9UOAMVBAwgYoFKlSoqKiir2+ddee63ef/991ahRQ8HBwec9p1atWtq4caM6duwoSTp79qy2bNmia6+99rznt2zZUoWFhUpJSVHXrl2LHD+XQBYUFDj3NWvWTHa7Xfv3779gcti0aVPnCy3nbNiw4dIPeRHffvutbrjhBj388MPOfenp6UXO27Ztm06fPu1sbjds2KCgoCDVqVNHYWFhl6wdALyFt4ABFHHvvfeqWrVq6tWrl7755htlZGRo9erVeuSRR/Trr79Kkh599FG98MILWrx4sXbt2qWHH374omv41atXT7Gxsbrvvvu0ePFi5zU/+OADSVJkZKRsNpuWLl2qI0eOKCcnR5UrV9bYsWM1evRozZs3T+np6dq6datmzJihefPmSZKGDx+uvXv3aty4cdq9e7cWLlyouXPnFus5f/vtN6Wmprpsx48fV6NGjbR582YtX75ce/bs0YQJE7Rp06Yi38/Pz9fQoUP1ww8/6LPPPtPEiRM1cuRI+fn5Fat2APAab09CBOBZf34JxJ3jmZmZ1uDBg61q1apZdrvdatCggTVs2DArOzvbsqw/Xvp49NFHreDgYCs0NNSKj4+3Bg8efMGXQCzLsk6fPm2NHj3aqlWrluXv729FRUVZb7/9tvN4YmKiFR4ebtlsNis2NtayrD9eXJk2bZrVpEkTq0KFClb16tWt7t27WykpKc7vffrpp1ZUVJRlt9utDh06WG+//XaxXgKRVGSbP3++lZeXZ8XFxVkhISFWaGio9dBDD1lPPPGE1apVqyJ/t2eeecaqWrWqFRQUZA0bNszKy8tznnOp2nkJBIC32CzrAjO2AQAAUCYxBAwAAGAYGkAAAADD0AACAAAYhgYQAADAMDSAAAAAhqEBBAAAMAwNIAAAgGFoAAEAAAxDAwgAAGAYGkAAAADD0AACAAAY5v8Db8GjDdx7+u4AAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    }
  ]
}