starter_script_task_1.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"1bKs7PJPTLA0kzEJk_GdiwHQEP9usC3vJ","timestamp":1669469142854}],"authorship_tag":"ABX9TyPO4BhaqeAD3t3pUARgJG3+"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"mXCk_QgGIkbJ","executionInfo":{"status":"ok","timestamp":1669553684737,"user_tz":0,"elapsed":17309,"user":{"displayName":"mael jullien","userId":"02909689638620016256"}},"outputId":"5751acb6-9338-4a89-9fd2-7630aca974b0"},"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting sklearn\n","  Downloading sklearn-0.0.post1.tar.gz (3.6 kB)\n","Building wheels for collected packages: sklearn\n","  Building wheel for sklearn (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for sklearn: filename=sklearn-0.0.post1-py3-none-any.whl size=2344 sha256=20d6aebc7e9b1f19c43145829ba82197dde1aadab5df234fa1e04d8c2e59a12d\n","  Stored in directory: /root/.cache/pip/wheels/42/56/cc/4a8bf86613aafd5b7f1b310477667c1fca5c51c3ae4124a003\n","Successfully built sklearn\n","Installing collected packages: sklearn\n","Successfully installed sklearn-0.0.post1\n"]}],"source":["!pip install sklearn\n","from sklearn.feature_extraction.text import TfidfVectorizer\n","from sklearn.metrics.pairwise import cosine_distances\n","import numpy\n","from sklearn.metrics import f1_score,precision_score,recall_score"]},{"cell_type":"markdown","source":["##1) Installations and imports\n"],"metadata":{"id":"t3nSCj8YJZXB"}},{"cell_type":"markdown","source":["a. Mount drive (if you are running on colab)"],"metadata":{"id":"ryi9cT9uJmMU"}},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZndzIUvHJaIK","executionInfo":{"status":"ok","timestamp":1669553704631,"user_tz":0,"elapsed":19898,"user":{"displayName":"mael jullien","userId":"02909689638620016256"}},"outputId":"f5e5b9a2-a931-4b7b-ae8a-e405cb382e59"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"markdown","source":["b. Clone or update competition repository\n","After cloning, under MyDrive, you will see NLI4CT-semeval-2023 folder with the training and dev set, aswell as the full list of CTRs."],"metadata":{"id":"83XnhyVqJbxA"}},{"cell_type":"code","source":["%cd /content/drive/MyDrive\n","\n","import os\n","\n","PROJECT_DIR = '/content/drive/MyDrive/NLI4CT-semeval-2023'\n","PROJECT_GITHUB_URL = 'https://github.com/ai-systems/nli4ct.git'\n","\n","if not os.path.isdir(PROJECT_DIR):\n","  !git clone {PROJECT_GITHUB_URL}\n","else:\n","  %cd {PROJECT_DIR}\n","  !git pull {PROJECT_GITHUB_URL}"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"hebTE3BrJ6v8","executionInfo":{"status":"ok","timestamp":1669553705023,"user_tz":0,"elapsed":400,"user":{"displayName":"mael jullien","userId":"02909689638620016256"}},"outputId":"dd13af90-f4cd-43d3-bf51-aca61a404746"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/drive/MyDrive\n","fatal: destination path 'nli4ct' already exists and is not an empty directory.\n"]}]},{"cell_type":"markdown","source":["##2) Dataset"],"metadata":{"id":"4sPSAOb5RXBD"}},{"cell_type":"code","source":["# Training data\n","!unzip /content/drive/MyDrive/nli4ct/training_data.zip"],"metadata":{"id":"GVtlF-bwMDL2","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669553717482,"user_tz":0,"elapsed":12463,"user":{"displayName":"mael jullien","userId":"02909689638620016256"}},"outputId":"67873d98-3553-4e9d-cf44-da9c1f2cf32d"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Archive:  /content/drive/MyDrive/nli4ct/training_data.zip\n","replace training_data/.DS_Store? [y]es, [n]o, [A]ll, [N]one, [r]ename: "]}]},{"cell_type":"code","source":["# Dev set\n","import json\n","\n","dev_path = \"/content/drive/MyDrive/training_data/dev.json\"\n","with open(dev_path) as json_file:\n","    dev = json.load(json_file)\n","\n","# Example instance\n","print(dev[list(dev.keys())[1]])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5DGqlGNyP0Be","executionInfo":{"status":"ok","timestamp":1669553764734,"user_tz":0,"elapsed":283,"user":{"displayName":"mael jullien","userId":"02909689638620016256"}},"outputId":"08f66589-011b-4af7-8f49-2da1fbf91df7"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["{'Type': 'Comparison', 'Section_id': 'Eligibility', 'Primary_id': 'NCT00425854', 'Secondary_id': 'NCT01224678', 'Statement': 'Patients with significantly elevated ejection fraction are excluded from the primary trial, but can still be eligible for the secondary trial if they are 55 years of age or over', 'Label': 'Contradiction', 'Primary_evidence_index': [15], 'Secondary_evidence_index': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]}\n"]}]},{"cell_type":"code","source":["uuid_list = list(dev.keys())\n","statements = []\n","gold_dev_primary_evidence = []\n","gold_dev_secondary_evidence = []\n","for i in range(len(uuid_list)):\n","  #Retrieve all statements from the development set\n","  statements.append(dev[uuid_list[i]][\"Statement\"])"],"metadata":{"id":"vGPNREODRohn"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["##3) TF-IDF Entailment prediction baseline"],"metadata":{"id":"CdGpPa-3RhUK"}},{"cell_type":"code","source":["Results = {}\n","\n","for i in range(len(uuid_list)):\n","  primary_ctr_path = os.path.join(\"/content/drive/MyDrive/training_data/CT json\",dev[uuid_list[i]][\"Primary_id\"]+\".json\")\n","  with open(primary_ctr_path) as json_file:\n","    primary_ctr = json.load(json_file)\n","  \n","  #Retrieve the full section from the primary trial\n","  primary_section = primary_ctr[dev[uuid_list[i]][\"Section_id\"]]\n","\n","  #Convert a primary section entries to a matrix of TF-IDF features.\n","  vectorizer = TfidfVectorizer().fit(primary_section)\n","  X_s = vectorizer.transform([statements[i]])\n","  X_p = vectorizer.transform(primary_section)\n","  #Compute the cosine similarity between the primary section entries and the statement\n","  primary_scores = cosine_distances(X_s, X_p)\n","  #Repeat for the secondary trial\n","  if dev[uuid_list[i]][\"Type\"] == \"Comparison\":\n","    secondary_ctr_path = os.path.join(\"/content/drive/MyDrive/training_data/CT json\",dev[uuid_list[i]][\"Secondary_id\"]+\".json\")\n","    with open(secondary_ctr_path) as json_file:\n","      secondary_ctr = json.load(json_file)\n","    secondary_section = secondary_ctr[dev[uuid_list[i]][\"Section_id\"]]\n","    vectorizer = TfidfVectorizer().fit(secondary_section)\n","    X_s = vectorizer.transform([statements[i]])\n","    X_p = vectorizer.transform(secondary_section)\n","    secondary_scores = cosine_distances(X_s, X_p)\n","    #Combine and average the cosine distances of all entries from the relevant section of the primary and secondary trial\n","    combined_scores = []\n","    combined_scores.extend(secondary_scores[0])\n","    combined_scores.extend(primary_scores[0])\n","    score = numpy.average(combined_scores)\n","    #If the cosine distance is gless than 0.9 the prediction is entailment   \n","    if score > 0.9:\n","      Prediction = \"Contradiction\"\n","    else:\n","      Prediction = \"Entailment\"\n","    Results[str(uuid_list[i])] = {\"Prediction\":Prediction}\n","  else:\n","    #If the cosine distance is greater than 0.9 the prediction is contradiction\n","    score = numpy.average(primary_scores)\n","    if score > 0.9:\n","      Prediction = \"Contradiction\"\n","    else:\n","      Prediction = \"Entailment\"\n","    Results[str(uuid_list[i])] = {\"Prediction\":Prediction}\n"],"metadata":{"id":"GLNvSMkhRgg2"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Save the results in the submission format."],"metadata":{"id":"_9_AfbVVvFt5"}},{"cell_type":"code","source":["print(Results)\n","with open(\"/content/drive/MyDrive/nli4ct/results.json\",'w') as jsonFile:\n","    jsonFile.write(json.dumps(Results,indent=4))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"W6CM2rNZpD7W","executionInfo":{"status":"ok","timestamp":1669553824706,"user_tz":0,"elapsed":375,"user":{"displayName":"mael jullien","userId":"02909689638620016256"}},"outputId":"38670791-5206-4672-9238-6e81c7f233db"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["{'1adc970c-d433-44d0-aa09-d3834986f7a2': {'Prediction': 'Contradiction'}, '6b9162d0-0816-46d4-81af-c60028dcc63b': {'Prediction': 'Entailment'}, '0b6cc8e3-69ee-4a91-b93d-2ad3fddce65f': {'Prediction': 'Contradiction'}, 'cc1f712a-2116-4e40-9810-f315e3fa5ff8': {'Prediction': 'Entailment'}, '904061c0-14fa-4f13-9118-9a41e24fa8eb': {'Prediction': 'Entailment'}, '43ee7645-ce1e-42d5-9a74-3e379f6f367b': {'Prediction': 'Contradiction'}, '0cef8c8e-7986-46c7-a597-c5733a9899c0': {'Prediction': 'Contradiction'}, '43ce26e5-03fa-4e9d-b0eb-6ea356295753': {'Prediction': 'Contradiction'}, '3facad41-0221-42f8-834d-470e65c4aad5': {'Prediction': 'Entailment'}, '9cbc00e9-3a2d-4471-a93e-72c95132fb6a': {'Prediction': 'Entailment'}, '8b91cab9-d858-45f3-bf8d-3d6fc55b4818': {'Prediction': 'Entailment'}, '4a75574c-fa86-4e62-a210-81c7b98a3807': {'Prediction': 'Contradiction'}, 'd0b50aeb-aad8-4a8d-aae6-5c58a7d382c7': {'Prediction': 'Entailment'}, 'b0b61978-57db-4a1c-812c-509e8b05f2dc': {'Prediction': 'Contradiction'}, '24b85b44-b9e6-4c28-b3aa-1bd97102b7f1': {'Prediction': 'Entailment'}, 'fbb62216-2cf2-4813-ad41-12a86940610f': {'Prediction': 'Contradiction'}, '961a956e-fece-4baa-a2ad-d3a5320bd9d2': {'Prediction': 'Contradiction'}, '97fd111c-6969-43ab-93af-b3dff09960e5': {'Prediction': 'Entailment'}, 'eb1bfb08-eba0-4130-b328-00f56c0f84a6': {'Prediction': 'Contradiction'}, 'b1219e71-fbf5-4bba-8a93-a27ae444e3e1': {'Prediction': 'Entailment'}, 'f10f3b84-e039-4a02-a98b-86d149d9a16f': {'Prediction': 'Contradiction'}, '0ddb07ae-6e70-436d-8723-f609e59c57da': {'Prediction': 'Entailment'}, 'c10eea1a-5768-442d-ae07-20b4f8f4b583': {'Prediction': 'Contradiction'}, '2ef186d0-b4ab-44de-b785-a63e9e95851a': {'Prediction': 'Contradiction'}, 'fef5aedb-21c0-4130-b8a3-cb1df5017c7a': {'Prediction': 'Entailment'}, '2655c892-216e-4b96-a8f2-6f64403215c5': {'Prediction': 'Contradiction'}, '2825d63b-8f22-4fb8-829f-9567845251c6': {'Prediction': 'Contradiction'}, '6b35bc10-f591-4b84-abac-40a74be949fa': {'Prediction': 'Entailment'}, 'a5baab2c-9c32-4301-a574-4ba5a0c2eb00': {'Prediction': 'Entailment'}, '70677d5f-7022-404c-8410-d1f89366d92f': {'Prediction': 'Contradiction'}, 'f46a7d27-a94f-4389-a1a0-4a8f5177b354': {'Prediction': 'Contradiction'}, 'fe876bb0-7ce3-4197-81ad-920f1f77938e': {'Prediction': 'Contradiction'}, 'f45ff82d-e0f5-456a-9d78-ddc2dc6ab929': {'Prediction': 'Contradiction'}, '4f705481-5dd0-414c-b375-c56e8c235b42': {'Prediction': 'Contradiction'}, '614d7ade-0fa7-4d77-bc19-1d3a2f2f48c2': {'Prediction': 'Entailment'}, 'c09fa61c-967a-40ae-96b0-c87b7cde237b': {'Prediction': 'Contradiction'}, '4309860d-042d-45a4-8a0a-a9fdd075247a': {'Prediction': 'Entailment'}, 'f2275929-7cfe-4bb1-a9f6-6bd0c4d5bf85': {'Prediction': 'Contradiction'}, '59cd7909-00c3-4c23-9a08-a42dbc8eabdd': {'Prediction': 'Entailment'}, 'e6a4e9a6-56b8-4a30-9743-eb02688c090f': {'Prediction': 'Contradiction'}, '82a3e542-f784-44d7-90f6-34d7e969283c': {'Prediction': 'Contradiction'}, 'fb360514-14be-45ab-8b74-ea5a7eb181d7': {'Prediction': 'Entailment'}, '7137dec6-5c98-4156-8035-7bbaaf0b649d': {'Prediction': 'Contradiction'}, '015dd869-db90-45a9-a951-9dc7dc2b2e01': {'Prediction': 'Entailment'}, '3a4003f6-ff1c-43d8-be48-f767266b2cf5': {'Prediction': 'Contradiction'}, '47ba01d6-edc7-4acb-ab49-b0e86f4eeaf7': {'Prediction': 'Entailment'}, 'f3cb9150-32c6-4877-9a87-4fac45a24948': {'Prediction': 'Entailment'}, '9d8b26d6-d574-4717-88b6-77458bc683f3': {'Prediction': 'Entailment'}, '51be7c95-8da4-419c-950c-91ebdd2c3fe5': {'Prediction': 'Entailment'}, '07042b0f-d422-49c3-8d53-81319927a4b3': {'Prediction': 'Entailment'}, '32361e21-ee57-4c34-9390-833ba5048a10': {'Prediction': 'Entailment'}, '88668508-dfd4-4a39-91cd-cf1654a6dfc4': {'Prediction': 'Entailment'}, 'dca4c9f7-bb2a-4f73-b390-93b7ced87c73': {'Prediction': 'Contradiction'}, 'e7604382-09cd-43b4-998f-a51b33f25c5a': {'Prediction': 'Contradiction'}, '75affd49-4905-4d43-9a5c-ef795281bac4': {'Prediction': 'Entailment'}, '2e76c74a-b6dd-47d8-a674-c1e80b0a3a65': {'Prediction': 'Entailment'}, '8c6314f7-34f5-42a7-ba66-b95d050a6315': {'Prediction': 'Contradiction'}, 'f114846e-79d8-4712-add0-bb34b65d418f': {'Prediction': 'Entailment'}, '812bd63c-4bfb-49cb-8fa9-baa3f8f6dabe': {'Prediction': 'Entailment'}, 'b1d28450-fc7d-496a-9c7c-bf711b28f8e4': {'Prediction': 'Entailment'}, 'c7dc582a-4a58-4cb5-b980-b2d2b6a128c3': {'Prediction': 'Entailment'}, 'a805f174-2707-4302-a195-05153ea8523e': {'Prediction': 'Contradiction'}, 'e4aeb013-7713-4b73-acbd-f8379cc6a494': {'Prediction': 'Contradiction'}, 'b8eaa8fd-61cf-4ca9-aaec-267d61c1fae3': {'Prediction': 'Entailment'}, '2729a321-709b-4392-9604-dd7237bedb17': {'Prediction': 'Contradiction'}, '8e9611fc-89f8-44cb-af09-8d71a13ca082': {'Prediction': 'Entailment'}, 'b9491012-49ba-4eb1-8230-1966f54ea012': {'Prediction': 'Contradiction'}, '62e449ad-16b3-479d-94a5-ffc435b5e99c': {'Prediction': 'Contradiction'}, '565d7982-aa9a-4c01-987c-1d67bc9c212d': {'Prediction': 'Contradiction'}, '7271e0db-bb82-423f-ad95-5ffa84fab97e': {'Prediction': 'Contradiction'}, '6402e13a-7cfb-4774-b7f9-80ecb79fda2a': {'Prediction': 'Entailment'}, '734423d5-4b47-4448-ab98-811015d626f0': {'Prediction': 'Entailment'}, 'e9fd065c-342c-4dfb-9c1c-a4c0c44d0acd': {'Prediction': 'Contradiction'}, '32ed503c-1cfd-44f6-b8bb-23d9d175d73d': {'Prediction': 'Contradiction'}, '175044aa-72b1-405d-8506-30002f5098e4': {'Prediction': 'Entailment'}, '045ad91e-44fa-42cc-ac22-a7aab26a9009': {'Prediction': 'Entailment'}, '00822101-fa21-4a26-bf45-3b1f535da005': {'Prediction': 'Contradiction'}, '06f798d1-2956-4903-83e9-58a41fa425f7': {'Prediction': 'Entailment'}, '8648c7df-582a-42a9-9330-414b2430ac56': {'Prediction': 'Contradiction'}, '89f610a7-9e81-43a4-8c1e-5503233b2be1': {'Prediction': 'Contradiction'}, 'b0046821-ba27-4a90-8869-cb708b843aa4': {'Prediction': 'Entailment'}, 'd1b217ae-76f3-4ba6-958e-5b2558703ba4': {'Prediction': 'Entailment'}, '8ddf8b88-6e06-4c71-9e28-45f73a637af0': {'Prediction': 'Entailment'}, '0ea29bd5-20be-40ea-8ad7-705d283d1d48': {'Prediction': 'Contradiction'}, 'e147796d-2edd-42d1-ac53-300fb8bd5eb6': {'Prediction': 'Entailment'}, '6b33b4ae-36f2-4a53-a954-6f246f6e023d': {'Prediction': 'Entailment'}, '06fad978-1dc3-46f5-b45f-5ac6577f28b9': {'Prediction': 'Entailment'}, '878c09e7-3fd9-404e-921e-5b549e9578fb': {'Prediction': 'Entailment'}, '3a0c517d-9b9a-4497-9675-50ff12cd02b7': {'Prediction': 'Entailment'}, '9abb10d7-496e-44fd-915d-d86aedf6c9b6': {'Prediction': 'Entailment'}, '8460a212-8054-4bd2-b4e6-af1d137a2f79': {'Prediction': 'Entailment'}, 'a3390ae6-d41d-4dc7-899e-de9534adfd6b': {'Prediction': 'Entailment'}, '17bbea92-8f92-41f3-9b9d-cf6e62cf6798': {'Prediction': 'Entailment'}, '794269ca-e7ec-4a65-ace5-b0f24a8cdc15': {'Prediction': 'Contradiction'}, '4dfff980-6a2b-4f01-9194-6ec831d6fed7': {'Prediction': 'Entailment'}, 'dc4c57b0-8c75-494e-81b0-3528df85ebff': {'Prediction': 'Contradiction'}, '8d9f94e3-871b-4fb4-a522-0a73d9d0ec3c': {'Prediction': 'Entailment'}, '251a20d8-2555-4929-9d01-c02311bdc93f': {'Prediction': 'Entailment'}, 'dd2f3c3d-fa95-4fca-a3f0-795fd4f3abf1': {'Prediction': 'Contradiction'}, 'f0273b8c-460d-46fe-9c52-8ff4b08e439b': {'Prediction': 'Contradiction'}, 'bc2024fc-5b39-4fa6-badf-f034d6fe8bb8': {'Prediction': 'Contradiction'}, 'c2217d42-e90c-4ffe-b084-e653b9ace4d5': {'Prediction': 'Entailment'}, '080da381-b5dd-46d3-99c4-18112da153c1': {'Prediction': 'Entailment'}, 'c100a7cd-e3a2-4901-a3e0-12336f769c7b': {'Prediction': 'Contradiction'}, 'ec1e269e-470d-43d9-8f23-8a19a5c96fcc': {'Prediction': 'Entailment'}, '21611c2a-e6eb-4012-a209-9b6781e1f260': {'Prediction': 'Entailment'}, 'd1b7105d-0b7d-4b10-92a7-d63243ff0629': {'Prediction': 'Entailment'}, '7f3fe97e-68ee-470f-b06e-21cdddb67aa8': {'Prediction': 'Contradiction'}, 'ce8464bd-951e-44aa-97d9-10b917b055ce': {'Prediction': 'Contradiction'}, '0131ebe3-746b-45a5-ba60-fdd326974fec': {'Prediction': 'Contradiction'}, 'b1dd7632-ed92-40a9-9293-bce68619c658': {'Prediction': 'Entailment'}, '45ccd2c8-6e02-4f0f-a370-eed18d26e6dc': {'Prediction': 'Entailment'}, '4a15ec5e-4904-4f01-a0a2-c51e83c8c070': {'Prediction': 'Entailment'}, 'b9e29ea8-6847-40d4-b5c3-81c496101695': {'Prediction': 'Entailment'}, '2a2c16a7-f427-4fdb-8153-94aac304e4b8': {'Prediction': 'Entailment'}, '618a3ae1-b22c-4c35-86a6-190c1b5b5d21': {'Prediction': 'Entailment'}, 'e62f5a3f-0091-4980-9822-db8f4dc2502a': {'Prediction': 'Entailment'}, '4a41ac73-538a-43e5-b7ce-8c8e2d56286e': {'Prediction': 'Contradiction'}, '75a65913-5655-4377-b441-ecdd4dd75175': {'Prediction': 'Entailment'}, '1329dec6-fb2c-4219-9db6-3fef9d94b316': {'Prediction': 'Entailment'}, '29b2fa29-5a76-4877-95bb-1a8de7973d33': {'Prediction': 'Entailment'}, '7edba90d-076d-4f55-b9d6-a1765ceb48bd': {'Prediction': 'Contradiction'}, 'ba9daeb9-796d-49ab-a30c-4d8acbdf4d37': {'Prediction': 'Entailment'}, '1314146f-2869-4525-8034-7f732d238385': {'Prediction': 'Contradiction'}, '7c7418d2-32d7-4517-874b-eb3db24ea16a': {'Prediction': 'Contradiction'}, '1f6a14ae-e22b-4b40-b389-217570dbf38e': {'Prediction': 'Contradiction'}, '5c1e360c-fd93-4dd4-b855-71ba416d34cb': {'Prediction': 'Contradiction'}, 'c7a953b0-ac96-4e0a-9c1d-98cf8af2861d': {'Prediction': 'Entailment'}, '48f516ff-abb4-4312-9b00-02f53fb16218': {'Prediction': 'Contradiction'}, '978d19fc-e7f6-485e-b8bd-cfcde06441ab': {'Prediction': 'Contradiction'}, '0a6d1b4c-244e-44e2-a229-62e4cbdfa979': {'Prediction': 'Contradiction'}, '9bcc31df-c1f6-4785-9235-f673213acf3a': {'Prediction': 'Entailment'}, '63a0a9fc-80b9-450f-b2c6-7aee37ce26a9': {'Prediction': 'Entailment'}, '75cf4ec2-9f2b-4d5b-8772-af247eaca07b': {'Prediction': 'Contradiction'}, '553f00ea-5197-4233-b1e0-e5727856b599': {'Prediction': 'Entailment'}, '57b2ae97-0d1d-4d5d-bc4b-109e8e5acb32': {'Prediction': 'Contradiction'}, '215a5c4a-d90c-4500-8284-0679ce311734': {'Prediction': 'Contradiction'}, 'e9a4bd91-f58b-4234-9037-75e0699d1bbd': {'Prediction': 'Entailment'}, '7d52734c-0801-413d-9ae2-d190b972cde5': {'Prediction': 'Contradiction'}, '5de9caeb-9dd5-4535-8fe3-289450d9c626': {'Prediction': 'Entailment'}, 'bebfc78e-b35f-4677-8342-9567a7e795a1': {'Prediction': 'Contradiction'}, 'faa65e54-aa04-4595-8dae-05cf00b4c887': {'Prediction': 'Entailment'}, '751fd763-89fa-4d52-8bda-f09b490a5e78': {'Prediction': 'Contradiction'}, 'c40fdf91-9f81-454a-bba9-ec5c950cef6e': {'Prediction': 'Entailment'}, '31105b43-832b-48bc-a78d-ee90c155ca44': {'Prediction': 'Entailment'}, '49ba06b2-576d-4db2-80b6-51673416f2ba': {'Prediction': 'Entailment'}, '3cb71095-8e01-40dc-bbf5-eeffed55f354': {'Prediction': 'Contradiction'}, '09fc4356-6154-427e-b710-ed5df6470aff': {'Prediction': 'Contradiction'}, '446f3c69-0de5-475c-a8e2-ff40ece90672': {'Prediction': 'Contradiction'}, 'cdadcb0a-20c5-47f7-a63a-39f5c6f26ae2': {'Prediction': 'Entailment'}, 'f0e845f9-8991-45b5-a520-51e934756f7a': {'Prediction': 'Entailment'}, 'a0065d58-e03b-4dd0-9c81-cbbf651a6d45': {'Prediction': 'Contradiction'}, 'b2cdbcb8-de26-4a44-8a94-dd68c13f7e41': {'Prediction': 'Contradiction'}, '973f090a-8967-4d75-bb8e-6166c868337c': {'Prediction': 'Entailment'}, '79ca2ee8-6c9c-40e6-a87c-925b50d3d3d2': {'Prediction': 'Entailment'}, '1d1f57f8-85f7-448c-a4bb-15957da568fd': {'Prediction': 'Entailment'}, '9f7d476e-9db3-4c00-abe6-4c2f0160470a': {'Prediction': 'Contradiction'}, 'a5e388b9-da3c-437e-a6af-7ba76ad8bf1d': {'Prediction': 'Entailment'}, '3adcc8fc-b76b-407a-a006-729a13f571fb': {'Prediction': 'Contradiction'}, '68379456-775b-463b-bfdb-d8aa42c9124b': {'Prediction': 'Entailment'}, '5d0ccb93-71ec-4079-92c6-d7cbf951a7bb': {'Prediction': 'Contradiction'}, 'f04d9695-81b8-4d70-877b-59caddf901bd': {'Prediction': 'Contradiction'}, 'f42e1874-343a-43ed-bb0c-884509c4793c': {'Prediction': 'Entailment'}, '30483d7e-7703-459c-bd10-e2bd259c2a4f': {'Prediction': 'Contradiction'}, '7cde9a22-a91f-4feb-892d-7cbd35f96a27': {'Prediction': 'Contradiction'}, '16c29502-4683-4087-9410-ff1b786556f5': {'Prediction': 'Contradiction'}, '5c975af8-6b7f-4a5d-b4a6-257c76763ce9': {'Prediction': 'Entailment'}, '1c4e3fe3-b5fe-4ea3-8d9e-0ce1502080e7': {'Prediction': 'Contradiction'}, '8e91e404-086e-4063-8d54-83d1b8ebd969': {'Prediction': 'Contradiction'}, 'b8322b23-a4f6-4cba-9761-be826b509de7': {'Prediction': 'Entailment'}, '4519d1cf-7f78-4aa8-85d6-ac94f7eb75a3': {'Prediction': 'Contradiction'}, '4b97a844-8d52-48bb-8eda-ca205268f016': {'Prediction': 'Entailment'}, '2a03c17f-3afe-40f1-a0db-4b876ee8c700': {'Prediction': 'Entailment'}, '3ba42f68-a643-4d10-afd0-cbb358d05a6f': {'Prediction': 'Entailment'}, '0a079cca-abde-42cf-9cec-a76e7a380e14': {'Prediction': 'Entailment'}, '9c033072-5f94-4d79-af47-126d531f0d6e': {'Prediction': 'Contradiction'}, 'a52b2d4f-0b06-4960-a9c3-88cb92df0980': {'Prediction': 'Entailment'}, 'a792bbac-0e8a-4bf7-9cc5-56fad481e2fb': {'Prediction': 'Contradiction'}, '1842904b-5371-49ee-b6f4-8831b879098b': {'Prediction': 'Entailment'}, '324562e6-2b61-4a07-b937-86f99689f2d5': {'Prediction': 'Contradiction'}, '74e376d7-5713-43f3-93f1-9608059c9547': {'Prediction': 'Entailment'}, '0e609fad-9a28-4fb6-90b6-32a731e3b02c': {'Prediction': 'Entailment'}, '236780db-9e59-4dc9-8da0-e786287011cf': {'Prediction': 'Entailment'}, '6d621974-c719-46df-8a50-d94e49583e33': {'Prediction': 'Entailment'}, 'a9907a2e-254a-45ca-af50-3e44fd2f7af3': {'Prediction': 'Entailment'}, '40f699a1-fbc6-40de-8e80-85b28f71632f': {'Prediction': 'Entailment'}, '6499b181-7a23-4073-8459-ad46ff02b20f': {'Prediction': 'Entailment'}, '2da13cd4-7189-4f85-be4d-5c1e213ae293': {'Prediction': 'Contradiction'}, 'a9bb6de9-c6f8-4c99-81cd-f101cbaab387': {'Prediction': 'Contradiction'}, '7e5fd062-6ec3-480f-ab0e-e117eba2ffe7': {'Prediction': 'Entailment'}, '016943fa-57ea-4293-8131-03292ea1836c': {'Prediction': 'Contradiction'}, '402f0dbf-cf67-4425-b300-a0a3df4d59d3': {'Prediction': 'Contradiction'}, '62be4676-099f-4c42-8029-773af8fde6f7': {'Prediction': 'Contradiction'}, '1180dd58-61b8-4c48-ab00-7d458e68c85b': {'Prediction': 'Contradiction'}, '566bdc2d-7c4a-4547-87da-ac81447ecca8': {'Prediction': 'Entailment'}, 'd310ec4e-993e-4827-8dc5-9aca053972db': {'Prediction': 'Entailment'}, 'a5617ae4-05a3-42d0-9e14-141de5f8c010': {'Prediction': 'Contradiction'}, '42d1fcd3-8faa-4065-bbba-42cc90ab67fb': {'Prediction': 'Contradiction'}, 'd01fda83-5dc8-4ad5-92b8-7553dabd7046': {'Prediction': 'Entailment'}, '985bc5a0-bc21-447f-92f1-150e23ede3f5': {'Prediction': 'Contradiction'}}\n"]}]},{"cell_type":"markdown","source":["##4) Evaluation"],"metadata":{"id":"CffKNPsaqPUK"}},{"cell_type":"markdown","source":["Run the task 1 evaluation script."],"metadata":{"id":"gdzgnw39u_TM"}},{"cell_type":"code","source":["def main():\n","\n","    gold = dev\n","    results = Results  \n","    uuid_list = list(results.keys())\n","\n","    results_pred = []\n","    gold_labels = []\n","    for i in range(len(uuid_list)):\n","        if results[uuid_list[i]][\"Prediction\"] == \"Entailment\":\n","            results_pred.append(1)\n","        else:\n","            results_pred.append(0)\n","        if gold[uuid_list[i]][\"Label\"] == \"Entailment\":\n","            gold_labels.append(1)\n","        else:\n","            gold_labels.append(0)\n","\n","    f_score = f1_score(gold_labels,results_pred)\n","    p_score = precision_score(gold_labels,results_pred)\n","    r_score = recall_score(gold_labels,results_pred)\n","\n","    print('F1:{:f}'.format(f_score))\n","    print('precision_score:{:f}'.format(p_score))\n","    print('recall_score:{:f}'.format(r_score))\n","\n","if '__main__' == __name__:\n","    main()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1JdL-lwitqgq","executionInfo":{"status":"ok","timestamp":1669553764454,"user_tz":0,"elapsed":11,"user":{"displayName":"mael jullien","userId":"02909689638620016256"}},"outputId":"b1a9dfe3-f7dd-42dd-a50b-f6278c740f8a"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["F1:0.502415\n","precision_score:0.485981\n","recall_score:0.520000\n"]}]}]}