From 71d92c5e5215d93817e0ebdb02f0080225222e4e Mon Sep 17 00:00:00 2001 From: Jithin James Date: Sat, 13 May 2023 04:04:23 +0530 Subject: [PATCH 01/22] some fixes --- belar/metrics/__init__.py | 5 ++--- belar/metrics/simple.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/belar/metrics/__init__.py b/belar/metrics/__init__.py index 8f6d9f56c..9131315af 100644 --- a/belar/metrics/__init__.py +++ b/belar/metrics/__init__.py @@ -1,5 +1,4 @@ from belar.metrics.base import Evaluation, Metric -from belar.metrics.similarity import * -from belar.metrics.simple import * -from belar.metrics.similarity import SBERTScore from belar.metrics.factual import EntailmentScore +from belar.metrics.similarity import SBERTScore +from belar.metrics.simple import * diff --git a/belar/metrics/simple.py b/belar/metrics/simple.py index 347b673dc..e2c83493d 100644 --- a/belar/metrics/simple.py +++ b/belar/metrics/simple.py @@ -1,8 +1,9 @@ from __future__ import annotations -from Levenshtein import distance, ratio import typing as t -from dataclasses import dataclass +from dataclasses import dataclass, field + +from Levenshtein import distance, ratio from nltk.tokenize import word_tokenize from nltk.translate.bleu_score import sentence_bleu from rouge_score import rouge_scorer @@ -14,7 +15,7 @@ @dataclass class BLEU(Metric): - weights: t.List[float] = [0.25, 0.25, 0.25, 0.25] + weights: list[float] = field(default_factory=lambda: [0.25, 0.25, 0.25, 0.25]) smoothing_function = None @property @@ -65,6 +66,7 @@ def score( return scores +@dataclass class EditScore(Metric): measure: t.Literal["distance", "ratio"] = "ratio" @@ -90,5 +92,8 @@ def score(self, ground_truth: t.List[str], generated_text: t.List[str]): Rouge1 = ROUGE("rouge1") Rouge2 = ROUGE("rouge2") RougeL = ROUGE("rougeL") +BLUE = BLEU() +EditDistance = EditScore("distance") +EditRatio = EditScore("ratio") -__all__ = ["Rouge1", "Rouge2", "RougeL"] +__all__ = ["Rouge1", "Rouge2", "RougeL", "BLEU", "EditDistance", "EditRatio"] From 63e81230b9f48180a161318707fd60f9f4181e2e Mon Sep 17 00:00:00 2001 From: Jithin James Date: Sat, 13 May 2023 04:06:44 +0530 Subject: [PATCH 02/22] add property to rouge --- belar/metrics/simple.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/belar/metrics/simple.py b/belar/metrics/simple.py index e2c83493d..882445a43 100644 --- a/belar/metrics/simple.py +++ b/belar/metrics/simple.py @@ -50,9 +50,11 @@ def __post_init__(self): [self.type], use_stemmer=self.use_stemmer ) + @property def name(self): return self.type + @property def is_batchable(self): return False From cd2ff2665962e33e0b442a9b0e5ff51a7547cda8 Mon Sep 17 00:00:00 2001 From: Jithin James Date: Sat, 13 May 2023 04:09:09 +0530 Subject: [PATCH 03/22] update quickstart --- examples/quickstart.ipynb | 855 ++++++++++++++++++++++++++++++-------- 1 file changed, 692 insertions(+), 163 deletions(-) diff --git a/examples/quickstart.ipynb b/examples/quickstart.ipynb index 6effae29b..43166fffe 100644 --- a/examples/quickstart.ipynb +++ b/examples/quickstart.ipynb @@ -2,8 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 7, - "id": "992c777a", + "execution_count": 1, + "id": "54b66a67", "metadata": {}, "outputs": [], "source": [ @@ -13,39 +13,10 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "5eaf4729", + "execution_count": null, + "id": "9710719d", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset eli5 (/home/jjmachan/.cache/huggingface/datasets/eli5/LFQA_reddit/1.0.0/17574e5502a10f41bbd17beba83e22475b499fa62caa1384a3d093fc856fe6fa)\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a2231863e61c4ffd8d695c8531a48139", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/9 [00:00 Date: Sat, 13 May 2023 04:09:29 +0530 Subject: [PATCH 04/22] add max_length --- belar/metrics/factual.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index fc41df3dd..4179f75dd 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -1,4 +1,5 @@ from __future__ import annotations +from tkinter.ttk import _Padding from transformers import AutoTokenizer, AutoModelForSequenceClassification import typing as t @@ -15,9 +16,11 @@ class EntailmentScore(Metric): """ model_name: str = "typeform/distilbert-base-uncased-mnli" + max_length: int = 512 batch_size: int = 4 device: t.Literal["cpu", "cuda"] = "cpu" + def __post_init__(self): self.device = device_check(self.device) self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) @@ -67,7 +70,8 @@ def score( """ encodings = self.tokenizer( - ground_truth, generated_text, truncation=True, return_tensors="pt" + ground_truth, generated_text, truncation=True, return_tensors="pt", + max_length=self.max_length, padding=True, ) score = self.batch_infer(encodings) From c282cc2aab767be794b7f05089e0f53b873bbeb9 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Sat, 13 May 2023 04:19:26 +0530 Subject: [PATCH 05/22] add max_length --- belar/metrics/factual.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 4179f75dd..6f630d138 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -1,5 +1,4 @@ from __future__ import annotations -from tkinter.ttk import _Padding from transformers import AutoTokenizer, AutoModelForSequenceClassification import typing as t @@ -71,7 +70,7 @@ def score( encodings = self.tokenizer( ground_truth, generated_text, truncation=True, return_tensors="pt", - max_length=self.max_length, padding=True, + max_length=self.max_length, padding="max_length", ) score = self.batch_infer(encodings) From 4fa981c1a42ef5a7ac16fead73b4671226d365cf Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Sat, 13 May 2023 20:13:57 +0530 Subject: [PATCH 06/22] add qa-qg paradigm --- belar/metrics/factual.py | 138 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 1 deletion(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 6f630d138..8f000b9eb 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -1,12 +1,20 @@ from __future__ import annotations +from textwrap import indent -from transformers import AutoTokenizer, AutoModelForSequenceClassification +import json +import numpy as np +import spacy +import torch +from transformers import AutoConfig, AutoTokenizer, AutoModelForSequenceClassification import typing as t from dataclasses import dataclass +import transformers + from belar.metrics import Metric from belar.utils import device_check +SPACY_MODEL = "en_core_web_sm" @dataclass class EntailmentScore(Metric): @@ -76,3 +84,131 @@ def score( score = self.batch_infer(encodings) return score + + +@dataclass +class Qsquare(Metric): + + qa_model_name:str = "" + qg_model_name: str = "" + device: t.Literal["cpu", "cuda"] = "cpu" + max_answers: int = 10 + candidate_filter: bool = True + load_single = False + batch_size: int = 4 + + + def __post_init__(self,): + + self.nlp = spacy.load(SPACY_MODEL) + self.qa = QAGQ.from_pretrained(self.qa_model_name) + self.qg = QAGQ.from_pretrained(self.qg_model_name) + + + + def generate_candidates(self, text:str): + + text = text.strip() + nouns = [i.text.lower() for i in self.nlp(text).noun_chunks if i.text.lower() not in self.nlp.Defaults.stop_words] + entities = [ent.text.lower() for ent in self.nlp(text)] + num_nouns = max(0, self.max_answers - len(entities)) + nouns = np.random.choice(np.setdiff1d(nouns, entities), size=num_nouns).tolist() + + return entities + nouns + + def generate_questions(self,candidates:list[str], context:str, **kwargs): + + questions = [] + for idx in range(0, len(candidates), self.batch_size): + batch_questions = self.qg.batch_generate_question(candidates[idx:idx+self.batch_size], + context, **kwargs) + questions.extend(batch_questions) + assert len(questions) == len(candidates), "Missing question for some candidates" + return questions + + def generate_answers(self, questions: list[str], context:str): + + answers = [] + for idx in range(0, len(questions), self.batch_size): + batch_answers = self.qa.batch_generate_answers(questions[idx:idx+self.batch_size],context) + answers.extend(batch_answers) + assert len(answers) == len(questions), "Missing answers for some questions" + return answers + + def filter_candidates(self, candidates:list[str],context:str): + + pass + + + def score(self, ground_truth: list[str], generated_text: list[str]): + + gnd_qans = {} + ans_candidates = [self.generate_candidates(text) for text in ground_truth] + for i,(candidates, context) in enumerate(zip(ans_candidates, ground_truth)): + questions = self.generate_questions(candidates, context, **kwargs) + gnd_qans[i] = [{"question": qstn, "answer":ans} for qstn, ans in zip(questions, candidates)] + + for i,gen_text in enumerate(generated_text): + questions = [item["question"] for item in gnd_qans[i]] + gen_answers = self.generate_answers(questions, gen_text) + gnd_qans[i] = [item.update({"predicted_answer":ans}) for item, ans in zip(gnd_qans[i], gen_answers)] + + with open("qa-qj-intermediate.json", "w") as file: + json.dump(gnd_qans, file, indent=4) + + + +from transformers.modeling_utils import PreTrainedModel +from transformers.models.auto.modeling_auto import ( + MODEL_WITH_LM_HEAD_MAPPING_NAMES, + MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES +) + +MODEL_MAPPINGS_NAMES = [MODEL_WITH_LM_HEAD_MAPPING_NAMES, MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES] + + +class QAGQ: + + def __init__(self,model:PreTrainedModel, model_name_or_path:str, device="cpu"): + + self.model = model.from_pretrained(model_name_or_path) + self.model.eval() + self.device = device_check(device) + self.model.to(self.device) + self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + + @classmethod + def from_pretrained(cls, model_name_or_path): + + config = AutoConfig.from_pretrained(model_name_or_path) + model_mappings = [arch for model_type in MODEL_MAPPINGS_NAMES for arch in model_type.values()] + architecture = np.intersect1d(model_mappings, config.architectures) + if len(architecture) == 0: + raise ValueError("Model doesn't support QA or LM architecture") + model = getattr(transformers, architecture[0]) + return cls(model, model_name_or_path) + + + def batch_generate_question(self,answers:list[str], contexts:str,**kwargs): + + input_texts = ["answer: %s context: %s " % (ans, context) for ans in answers] + max_length = kwargs.pop("input_max_length",512) + encodings = self.tokenizer(input_texts, padding="max_length", truncation=True, + max_length = max_length, return_tensors="pt") + encodings = {k:v.to(self.device) for k,v in encodings.items()} + outputs = self.model.generate(**encodings,**kwargs) + outputs = self.tokenizer.batch_decode(outputs,skip_special_tokens=True) + return [output.replace("question:","").strip() for output in outputs] + + def batch_generate_answers(self, questions:list[str], context:str, **kwargs): + + max_length = kwargs.pop("input_max_length",512) + encodings = self.tokenizer(questions, [context] * len(questions), padding="max_length", truncation=True, + max_length = max_length, return_tensors="pt") + encodings = {k:v.view(-1,max_length).to(self.device) for k,v in encodings.items()} + poss_ans_starts, poss_ans_ends = self.model(**encodings, return_dict=False) + best_start = poss_ans_starts.argmax(1) + best_ends = poss_ans_ends.argmax(1) + answers = [encodings["input_ids"][i][start:end+1] for i, (start, end) in enumerate(zip(best_start, best_ends))] + answers = self.tokenizer.batch_decode(answers) + return answers From 11bdb5062ae4e87a761c48a7313333efacddc4e6 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sat, 13 May 2023 17:23:50 +0000 Subject: [PATCH 07/22] fix model pipeline --- belar/metrics/factual.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 8f000b9eb..98e8bec58 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -104,17 +104,27 @@ def __post_init__(self,): self.qa = QAGQ.from_pretrained(self.qa_model_name) self.qg = QAGQ.from_pretrained(self.qg_model_name) - + @property + def name(self): + return "Q^2" + + @property + def is_batchable(self): + return True def generate_candidates(self, text:str): text = text.strip() nouns = [i.text.lower() for i in self.nlp(text).noun_chunks if i.text.lower() not in self.nlp.Defaults.stop_words] - entities = [ent.text.lower() for ent in self.nlp(text)] + entities = [ent.text.lower() for ent in self.nlp(text).ents] num_nouns = max(0, self.max_answers - len(entities)) - nouns = np.random.choice(np.setdiff1d(nouns, entities), size=num_nouns).tolist() + nouns = list(np.setdiff1d(nouns, entities)) + if nouns: + nouns = np.random.choice(nouns, size=num_nouns).tolist() + else: + nouns = [] - return entities + nouns + return entities + list(set(nouns)) def generate_questions(self,candidates:list[str], context:str, **kwargs): @@ -140,7 +150,7 @@ def filter_candidates(self, candidates:list[str],context:str): pass - def score(self, ground_truth: list[str], generated_text: list[str]): + def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): gnd_qans = {} ans_candidates = [self.generate_candidates(text) for text in ground_truth] @@ -151,10 +161,11 @@ def score(self, ground_truth: list[str], generated_text: list[str]): for i,gen_text in enumerate(generated_text): questions = [item["question"] for item in gnd_qans[i]] gen_answers = self.generate_answers(questions, gen_text) - gnd_qans[i] = [item.update({"predicted_answer":ans}) for item, ans in zip(gnd_qans[i], gen_answers)] + _ = [item.update({"predicted_answer":ans}) for item, ans in zip(gnd_qans[i], gen_answers)] with open("qa-qj-intermediate.json", "w") as file: json.dump(gnd_qans, file, indent=4) + return gnd_qans @@ -189,7 +200,7 @@ def from_pretrained(cls, model_name_or_path): return cls(model, model_name_or_path) - def batch_generate_question(self,answers:list[str], contexts:str,**kwargs): + def batch_generate_question(self,answers:list[str], context:str,**kwargs): input_texts = ["answer: %s context: %s " % (ans, context) for ans in answers] max_length = kwargs.pop("input_max_length",512) From b799c2d4bdc120da5343f0d0bc482deea0a394ef Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 05:17:15 +0000 Subject: [PATCH 08/22] add filter candidates --- belar/metrics/factual.py | 47 +++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 98e8bec58..516c4f996 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -1,5 +1,4 @@ from __future__ import annotations -from textwrap import indent import json import numpy as np @@ -96,6 +95,7 @@ class Qsquare(Metric): candidate_filter: bool = True load_single = False batch_size: int = 4 + include_nouns: bool = False def __post_init__(self,): @@ -116,15 +116,15 @@ def generate_candidates(self, text:str): text = text.strip() nouns = [i.text.lower() for i in self.nlp(text).noun_chunks if i.text.lower() not in self.nlp.Defaults.stop_words] - entities = [ent.text.lower() for ent in self.nlp(text).ents] + entities = set([ent.text.lower() for ent in self.nlp(text).ents]) num_nouns = max(0, self.max_answers - len(entities)) - nouns = list(np.setdiff1d(nouns, entities)) - if nouns: + nouns = list(np.setdiff1d(nouns, list(entities))) + if nouns and self.include_nouns: nouns = np.random.choice(nouns, size=num_nouns).tolist() else: nouns = [] - return entities + list(set(nouns)) + return list(entities.union(set(nouns))) def generate_questions(self,candidates:list[str], context:str, **kwargs): @@ -145,17 +145,46 @@ def generate_answers(self, questions: list[str], context:str): assert len(answers) == len(questions), "Missing answers for some questions" return answers - def filter_candidates(self, candidates:list[str],context:str): + def filter_candidates(self, questions:list[str], candidates:list[str], gen_answers:list[str]): + final_questions = [] + final_candidates = [] + for qstn, ans1, ans2 in zip(questions, candidates, gen_answers): + if self.clean_candidate(ans1) == self.clean_candidate(ans2): + final_candidates.append(ans1) + final_questions.append(qstn) + + return final_questions, final_candidates + + + def clean_candidate(self, text): pass + #strip-lower-remove punct- + + def score_candidates(self, ques_ans_dict:dict): + + nli = EntailmentScore() + for qas in ques_ans_dict.values(): + for item in qas: + item["answer"] = self.clean_candidate(item["answer"]) + item["predicted_answer"] = self.clean_candidate(item["predicted_answer"]) + if item["answer"] == item["predicted_answer"]: + item.update({"score":1}) + else: + score = nli.score([item.get("answer")],[item.get("predicted_answer")]) + item.update({"score":score}) + return ques_ans_dict + def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): gnd_qans = {} ans_candidates = [self.generate_candidates(text) for text in ground_truth] for i,(candidates, context) in enumerate(zip(ans_candidates, ground_truth)): questions = self.generate_questions(candidates, context, **kwargs) + gen_answers = self.generate_answers(questions, context) + questions, candidates = self.filter_candidates(questions, candidates, gen_answers) gnd_qans[i] = [{"question": qstn, "answer":ans} for qstn, ans in zip(questions, candidates)] for i,gen_text in enumerate(generated_text): @@ -165,7 +194,13 @@ def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): with open("qa-qj-intermediate.json", "w") as file: json.dump(gnd_qans, file, indent=4) + + gnd_gans = self.score_candidates(gnd_qans) + return gnd_qans + + + From e4e773d327496b7aa78cbad1086c6a839f277cf5 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 05:42:18 +0000 Subject: [PATCH 09/22] add candidate cleaning --- belar/metrics/factual.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 516c4f996..a427c545f 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -2,6 +2,8 @@ import json import numpy as np +import re +import string import spacy import torch from transformers import AutoConfig, AutoTokenizer, AutoModelForSequenceClassification @@ -95,7 +97,7 @@ class Qsquare(Metric): candidate_filter: bool = True load_single = False batch_size: int = 4 - include_nouns: bool = False + include_nouns: bool = True def __post_init__(self,): @@ -132,7 +134,7 @@ def generate_questions(self,candidates:list[str], context:str, **kwargs): for idx in range(0, len(candidates), self.batch_size): batch_questions = self.qg.batch_generate_question(candidates[idx:idx+self.batch_size], context, **kwargs) - questions.extend(batch_questions) + questions.extend([qstn if qstn.endswith('?') else f'{qstn}?' for qstn in batch_questions ]) assert len(questions) == len(candidates), "Missing question for some candidates" return questions @@ -158,10 +160,13 @@ def filter_candidates(self, questions:list[str], candidates:list[str], gen_answe def clean_candidate(self, text): - pass + #strip-lower-remove punct- + text = text.strip().lower() + text = text.translate(str.maketrans('', '', string.punctuation)) + text = re.sub(r'\b(a|an|the|in|our)\b', ' ', text) - + return text def score_candidates(self, ques_ans_dict:dict): nli = EntailmentScore() @@ -172,7 +177,9 @@ def score_candidates(self, ques_ans_dict:dict): if item["answer"] == item["predicted_answer"]: item.update({"score":1}) else: - score = nli.score([item.get("answer")],[item.get("predicted_answer")]) + qstn = item.get("question") + score = nli.score([f'{qstn}{item.get("answer")}'], + [f'{qstn}{item.get("predicted_answer")}']) item.update({"score":score}) return ques_ans_dict @@ -195,7 +202,13 @@ def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): with open("qa-qj-intermediate.json", "w") as file: json.dump(gnd_qans, file, indent=4) - gnd_gans = self.score_candidates(gnd_qans) + del self.qa + del self.qg + + gnd_qans = self.score_candidates(gnd_qans) + + with open("qa-qj-intermediate.json", "w") as file: + json.dump(gnd_qans, file, indent=4) return gnd_qans From 6bf4e4cd6292a23b9df4adfed5321019df4becd6 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 09:08:39 +0000 Subject: [PATCH 10/22] compute scores from qa-qj --- belar/metrics/factual.py | 283 +++++++++++++++++++++++---------------- 1 file changed, 168 insertions(+), 115 deletions(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index a427c545f..de258f755 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -5,17 +5,32 @@ import re import string import spacy -import torch +import transformers from transformers import AutoConfig, AutoTokenizer, AutoModelForSequenceClassification +from transformers.modeling_utils import PreTrainedModel import typing as t from dataclasses import dataclass -import transformers from belar.metrics import Metric from belar.utils import device_check + +from transformers.models.auto.modeling_auto import ( + MODEL_WITH_LM_HEAD_MAPPING_NAMES, + MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES, +) + +MODEL_MAPPINGS_NAMES = [ + MODEL_WITH_LM_HEAD_MAPPING_NAMES, + MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES, +] + + SPACY_MODEL = "en_core_web_sm" +LABEL2SCORE = {"entailment": 1, "contradiction": 0, "neutral": 0} +EPS = 1e-8 + @dataclass class EntailmentScore(Metric): @@ -23,12 +38,11 @@ class EntailmentScore(Metric): Entailment score using ground truth as premise and generated text as hypothesis. """ - model_name: str = "typeform/distilbert-base-uncased-mnli" + model_name: str = "cross-encoder/nli-roberta-base" max_length: int = 512 batch_size: int = 4 device: t.Literal["cpu", "cuda"] = "cpu" - def __post_init__(self): self.device = device_check(self.device) self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) @@ -53,6 +67,20 @@ def name(self): def is_batchable(self): return True + def infer(self, ground_truth: str, generated_text: str): + encodings = self.tokenizer( + ground_truth, + generated_text, + truncation=True, + return_tensors="pt", + max_length=self.max_length, + padding="max_length", + ) + label2id = {value.lower(): key for key, value in self.id2label.items()} + output = self.model(**encodings) + pred = output.logits.softmax(axis=-1).detach().cpu().squeeze() + return {label: pred[id].item() for label, id in label2id.items()} + def batch_infer(self, inputs: dict): predictions = [] input_ids = inputs["input_ids"] @@ -78,8 +106,12 @@ def score( """ encodings = self.tokenizer( - ground_truth, generated_text, truncation=True, return_tensors="pt", - max_length=self.max_length, padding="max_length", + ground_truth, + generated_text, + truncation=True, + return_tensors="pt", + max_length=self.max_length, + padding="max_length", ) score = self.batch_infer(encodings) @@ -87,37 +119,101 @@ def score( return score +class QAGQ: + def __init__(self, model: PreTrainedModel, model_name_or_path: str, device="cpu"): + self.model = model.from_pretrained(model_name_or_path) + self.model.eval() + self.device = device_check(device) + self.model.to(self.device) + self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + + @classmethod + def from_pretrained(cls, model_name_or_path): + config = AutoConfig.from_pretrained(model_name_or_path) + model_mappings = [ + arch for model_type in MODEL_MAPPINGS_NAMES for arch in model_type.values() + ] + architecture = np.intersect1d(model_mappings, config.architectures) + if len(architecture) == 0: + raise ValueError("Model doesn't support QA or LM architecture") + model = getattr(transformers, architecture[0]) + return cls(model, model_name_or_path) + + def batch_generate_question(self, answers: list[str], context: str, **kwargs): + input_texts = [ + "answer: %s context: %s " % (ans, context) for ans in answers + ] + max_length = kwargs.pop("input_max_length", 512) + encodings = self.tokenizer( + input_texts, + padding="max_length", + truncation=True, + max_length=max_length, + return_tensors="pt", + ) + encodings = {k: v.to(self.device) for k, v in encodings.items()} + outputs = self.model.generate(**encodings, **kwargs) + outputs = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) + return [output.replace("question:", "").strip() for output in outputs] + + def batch_generate_answers(self, questions: list[str], context: str, **kwargs): + max_length = kwargs.pop("input_max_length", 512) + encodings = self.tokenizer( + questions, + [context] * len(questions), + padding="max_length", + truncation=True, + max_length=max_length, + return_tensors="pt", + ) + encodings = { + k: v.view(-1, max_length).to(self.device) for k, v in encodings.items() + } + poss_ans_starts, poss_ans_ends = self.model(**encodings, return_dict=False) + best_start = poss_ans_starts.argmax(1) + best_ends = poss_ans_ends.argmax(1) + answers = [ + encodings["input_ids"][i][start : end + 1] + for i, (start, end) in enumerate(zip(best_start, best_ends)) + ] + answers = self.tokenizer.batch_decode(answers) + return answers + + @dataclass class Qsquare(Metric): - - qa_model_name:str = "" - qg_model_name: str = "" + qa_model_name: str = "consciousAI/question-answering-roberta-base-s" + qg_model_name: str = "mrm8488/t5-base-finetuned-question-generation-ap" device: t.Literal["cpu", "cuda"] = "cpu" max_answers: int = 10 - candidate_filter: bool = True + crosscheck_candidates: bool = True load_single = False batch_size: int = 4 include_nouns: bool = True + save_intermediate = False - - def __post_init__(self,): - + def __post_init__( + self, + ): self.nlp = spacy.load(SPACY_MODEL) self.qa = QAGQ.from_pretrained(self.qa_model_name) self.qg = QAGQ.from_pretrained(self.qg_model_name) - + @property def name(self): return "Q^2" - + @property def is_batchable(self): return True - def generate_candidates(self, text:str): - + def generate_candidates(self, text: str): text = text.strip() - nouns = [i.text.lower() for i in self.nlp(text).noun_chunks if i.text.lower() not in self.nlp.Defaults.stop_words] + nouns = [ + i.text.lower() + for i in self.nlp(text).noun_chunks + if i.text.lower() not in self.nlp.Defaults.stop_words + ] entities = set([ent.text.lower() for ent in self.nlp(text).ents]) num_nouns = max(0, self.max_answers - len(entities)) nouns = list(np.setdiff1d(nouns, list(entities))) @@ -127,147 +223,104 @@ def generate_candidates(self, text:str): nouns = [] return list(entities.union(set(nouns))) - - def generate_questions(self,candidates:list[str], context:str, **kwargs): + def generate_questions(self, candidates: list[str], context: str, **kwargs): questions = [] for idx in range(0, len(candidates), self.batch_size): - batch_questions = self.qg.batch_generate_question(candidates[idx:idx+self.batch_size], - context, **kwargs) - questions.extend([qstn if qstn.endswith('?') else f'{qstn}?' for qstn in batch_questions ]) + batch_questions = self.qg.batch_generate_question( + candidates[idx : idx + self.batch_size], context, **kwargs + ) + questions.extend( + [qstn if qstn.endswith("?") else f"{qstn}?" for qstn in batch_questions] + ) assert len(questions) == len(candidates), "Missing question for some candidates" return questions - def generate_answers(self, questions: list[str], context:str): - + def generate_answers(self, questions: list[str], context: str): answers = [] for idx in range(0, len(questions), self.batch_size): - batch_answers = self.qa.batch_generate_answers(questions[idx:idx+self.batch_size],context) + batch_answers = self.qa.batch_generate_answers( + questions[idx : idx + self.batch_size], context + ) answers.extend(batch_answers) assert len(answers) == len(questions), "Missing answers for some questions" return answers - def filter_candidates(self, questions:list[str], candidates:list[str], gen_answers:list[str]): - + def filter_candidates( + self, questions: list[str], candidates: list[str], gen_answers: list[str] + ): final_questions = [] final_candidates = [] for qstn, ans1, ans2 in zip(questions, candidates, gen_answers): if self.clean_candidate(ans1) == self.clean_candidate(ans2): final_candidates.append(ans1) final_questions.append(qstn) - + return final_questions, final_candidates - def clean_candidate(self, text): - - #strip-lower-remove punct- text = text.strip().lower() - text = text.translate(str.maketrans('', '', string.punctuation)) - text = re.sub(r'\b(a|an|the|in|our)\b', ' ', text) + text = text.translate(str.maketrans("", "", string.punctuation)) + text = re.sub(r"\b(a|an|the|in|our)\b", " ", text) return text - def score_candidates(self, ques_ans_dict:dict): - + + def score_candidates(self, ques_ans_dict: dict): nli = EntailmentScore() for qas in ques_ans_dict.values(): for item in qas: item["answer"] = self.clean_candidate(item["answer"]) - item["predicted_answer"] = self.clean_candidate(item["predicted_answer"]) + item["predicted_answer"] = self.clean_candidate( + item["predicted_answer"] + ) if item["answer"] == item["predicted_answer"]: - item.update({"score":1}) + item.update({"score": 1}) else: qstn = item.get("question") - score = nli.score([f'{qstn}{item.get("answer")}'], - [f'{qstn}{item.get("predicted_answer")}']) - item.update({"score":score}) - + score_dict = nli.infer( + f'{qstn}{item.get("answer")}', + f'{qstn}{item.get("predicted_answer")}', + ) + label = max(zip(score_dict.values(), score_dict.keys()))[1] + item.update({"score": LABEL2SCORE[label]}) + return ques_ans_dict - + def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): - gnd_qans = {} ans_candidates = [self.generate_candidates(text) for text in ground_truth] - for i,(candidates, context) in enumerate(zip(ans_candidates, ground_truth)): + for i, (candidates, context) in enumerate(zip(ans_candidates, ground_truth)): questions = self.generate_questions(candidates, context, **kwargs) gen_answers = self.generate_answers(questions, context) - questions, candidates = self.filter_candidates(questions, candidates, gen_answers) - gnd_qans[i] = [{"question": qstn, "answer":ans} for qstn, ans in zip(questions, candidates)] - - for i,gen_text in enumerate(generated_text): + if self.crosscheck_candidates: + questions, candidates = self.filter_candidates( + questions, candidates, gen_answers + ) + gnd_qans[i] = [ + {"question": qstn, "answer": ans} + for qstn, ans in zip(questions, candidates) + ] + + for i, gen_text in enumerate(generated_text): questions = [item["question"] for item in gnd_qans[i]] gen_answers = self.generate_answers(questions, gen_text) - _ = [item.update({"predicted_answer":ans}) for item, ans in zip(gnd_qans[i], gen_answers)] + _ = [ + item.update({"predicted_answer": ans}) + for item, ans in zip(gnd_qans[i], gen_answers) + ] with open("qa-qj-intermediate.json", "w") as file: json.dump(gnd_qans, file, indent=4) - + del self.qa del self.qg - - gnd_qans = self.score_candidates(gnd_qans) - - with open("qa-qj-intermediate.json", "w") as file: - json.dump(gnd_qans, file, indent=4) - - return gnd_qans - - - - - - -from transformers.modeling_utils import PreTrainedModel -from transformers.models.auto.modeling_auto import ( - MODEL_WITH_LM_HEAD_MAPPING_NAMES, - MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES -) - -MODEL_MAPPINGS_NAMES = [MODEL_WITH_LM_HEAD_MAPPING_NAMES, MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES] + gnd_qans = self.score_candidates(gnd_qans) -class QAGQ: - - def __init__(self,model:PreTrainedModel, model_name_or_path:str, device="cpu"): - - self.model = model.from_pretrained(model_name_or_path) - self.model.eval() - self.device = device_check(device) - self.model.to(self.device) - self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) - - @classmethod - def from_pretrained(cls, model_name_or_path): - - config = AutoConfig.from_pretrained(model_name_or_path) - model_mappings = [arch for model_type in MODEL_MAPPINGS_NAMES for arch in model_type.values()] - architecture = np.intersect1d(model_mappings, config.architectures) - if len(architecture) == 0: - raise ValueError("Model doesn't support QA or LM architecture") - model = getattr(transformers, architecture[0]) - return cls(model, model_name_or_path) + if self.save_intermediate: + with open("qa-qj-intermediate.json", "w") as file: + json.dump(gnd_qans, file, indent=4) - - def batch_generate_question(self,answers:list[str], context:str,**kwargs): - - input_texts = ["answer: %s context: %s " % (ans, context) for ans in answers] - max_length = kwargs.pop("input_max_length",512) - encodings = self.tokenizer(input_texts, padding="max_length", truncation=True, - max_length = max_length, return_tensors="pt") - encodings = {k:v.to(self.device) for k,v in encodings.items()} - outputs = self.model.generate(**encodings,**kwargs) - outputs = self.tokenizer.batch_decode(outputs,skip_special_tokens=True) - return [output.replace("question:","").strip() for output in outputs] - - def batch_generate_answers(self, questions:list[str], context:str, **kwargs): - - max_length = kwargs.pop("input_max_length",512) - encodings = self.tokenizer(questions, [context] * len(questions), padding="max_length", truncation=True, - max_length = max_length, return_tensors="pt") - encodings = {k:v.view(-1,max_length).to(self.device) for k,v in encodings.items()} - poss_ans_starts, poss_ans_ends = self.model(**encodings, return_dict=False) - best_start = poss_ans_starts.argmax(1) - best_ends = poss_ans_ends.argmax(1) - answers = [encodings["input_ids"][i][start:end+1] for i, (start, end) in enumerate(zip(best_start, best_ends))] - answers = self.tokenizer.batch_decode(answers) - return answers + scores = [[dic["score"] for dic in item] for item in gnd_qans.values()] + scores = [sum(sublist) / (len(sublist) + EPS) for sublist in scores] + return scores From 8899304a4b1996580d4ac0a7b7a23ffdcf328fc6 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 09:11:50 +0000 Subject: [PATCH 11/22] remove json saving --- belar/metrics/factual.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index de258f755..ad92fdcb0 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -308,10 +308,7 @@ def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): item.update({"predicted_answer": ans}) for item, ans in zip(gnd_qans[i], gen_answers) ] - - with open("qa-qj-intermediate.json", "w") as file: - json.dump(gnd_qans, file, indent=4) - + del self.qa del self.qg @@ -324,3 +321,5 @@ def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): scores = [[dic["score"] for dic in item] for item in gnd_qans.values()] scores = [sum(sublist) / (len(sublist) + EPS) for sublist in scores] return scores + +__all__ = ["EntailmentScore","Qsquare"] From 908b2a038893cadf00f8b1da1edfdb6084972fc0 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 12:52:11 +0000 Subject: [PATCH 12/22] change score for neutral --- belar/metrics/factual.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index ad92fdcb0..3c80719e9 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -28,7 +28,7 @@ SPACY_MODEL = "en_core_web_sm" -LABEL2SCORE = {"entailment": 1, "contradiction": 0, "neutral": 0} +LABEL2SCORE = {"entailment": 1, "contradiction": 0, "neutral": 0.5} EPS = 1e-8 @@ -190,7 +190,7 @@ class Qsquare(Metric): load_single = False batch_size: int = 4 include_nouns: bool = True - save_intermediate = False + save_results: bool = False def __post_init__( self, @@ -314,7 +314,7 @@ def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): gnd_qans = self.score_candidates(gnd_qans) - if self.save_intermediate: + if self.save_results: with open("qa-qj-intermediate.json", "w") as file: json.dump(gnd_qans, file, indent=4) From 803d13778225439e1cf9d32aafade300748145d9 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 13:04:57 +0000 Subject: [PATCH 13/22] resolve merge conflicts --- belar/metrics/__init__.py | 3 +-- belar/metrics/factual.py | 5 +++-- belar/metrics/simple.py | 8 -------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/belar/metrics/__init__.py b/belar/metrics/__init__.py index 70fe4a3e5..e9bef643f 100644 --- a/belar/metrics/__init__.py +++ b/belar/metrics/__init__.py @@ -1,8 +1,7 @@ from belar.metrics.base import Evaluation, Metric from belar.metrics.factual import EntailmentScore from belar.metrics.similarity import SBERTScore -from belar.metrics.simple import (BLUE, EditDistance, EditRatio, Rouge1, - Rouge2, RougeL) +from belar.metrics.simple import BLUE, EditDistance, EditRatio, Rouge1, Rouge2, RougeL __all__ = [ "Evaluation", diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 0e5ef3388..c99f58297 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -304,7 +304,7 @@ def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): item.update({"predicted_answer": ans}) for item, ans in zip(gnd_qans[i], gen_answers) ] - + del self.qa del self.qg @@ -318,4 +318,5 @@ def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): scores = [sum(sublist) / (len(sublist) + EPS) for sublist in scores] return scores -__all__ = ["EntailmentScore","Qsquare"] + +__all__ = ["EntailmentScore", "Qsquare"] diff --git a/belar/metrics/simple.py b/belar/metrics/simple.py index 9b1377ac3..116225776 100644 --- a/belar/metrics/simple.py +++ b/belar/metrics/simple.py @@ -94,14 +94,6 @@ def score(self, ground_truth: t.List[str], generated_text: t.List[str]): Rouge1 = ROUGE("rouge1") Rouge2 = ROUGE("rouge2") RougeL = ROUGE("rougeL") -<<<<<<< HEAD -BLUE = BLEU() -EditDistance = EditScore("distance") -EditRatio = EditScore("ratio") - -__all__ = ["Rouge1", "Rouge2", "RougeL", "BLEU", "EditDistance", "EditRatio"] -======= BLUE = BLEUScore() EditDistance = EditScore("distance") EditRatio = EditScore("ratio") ->>>>>>> main From d9661bbee6558a789eab0e40b6e7fde571475c63 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 13:05:21 +0000 Subject: [PATCH 14/22] resolve merge conflicts --- tests/benchmarks/benchmark.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/benchmarks/benchmark.py b/tests/benchmarks/benchmark.py index 50d63beb5..12732828b 100644 --- a/tests/benchmarks/benchmark.py +++ b/tests/benchmarks/benchmark.py @@ -5,8 +5,16 @@ from tqdm import tqdm from utils import print_table, timeit -from belar.metrics import (EditDistance, EditRatio, EntailmentScore, - Evaluation, Rouge1, Rouge2, RougeL, SBERTScore) +from belar.metrics import ( + EditDistance, + EditRatio, + EntailmentScore, + Evaluation, + Rouge1, + Rouge2, + RougeL, + SBERTScore, +) DEVICE = "cuda" if is_available() else "cpu" BATCHES = [0, 1] From 2ac5bcbd25ab475e575f3fff9a568764bb718c07 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 13:10:00 +0000 Subject: [PATCH 15/22] initialize metric instances --- belar/metrics/factual.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index c99f58297..4ec7b1f02 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -318,5 +318,5 @@ def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): scores = [sum(sublist) / (len(sublist) + EPS) for sublist in scores] return scores - -__all__ = ["EntailmentScore", "Qsquare"] +ENTScore = EntailmentScore() +Q2Score = Qsquare() \ No newline at end of file From f0a1f62b4c86c437d72a2f0cec064cf88795a080 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 13:10:32 +0000 Subject: [PATCH 16/22] black reformatting --- belar/metrics/factual.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 4ec7b1f02..7f182063a 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -318,5 +318,6 @@ def score(self, ground_truth: list[str], generated_text: list[str], **kwargs): scores = [sum(sublist) / (len(sublist) + EPS) for sublist in scores] return scores + ENTScore = EntailmentScore() -Q2Score = Qsquare() \ No newline at end of file +Q2Score = Qsquare() From 7f01ddc523b2eeac571533b6b7eb52532f0a82b6 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 13:29:19 +0000 Subject: [PATCH 17/22] format quickstart --- examples/belar-test.ipynb | 544 +++++++++++++++++++++++++ examples/quickstart.ipynb | 827 +------------------------------------- 2 files changed, 548 insertions(+), 823 deletions(-) create mode 100644 examples/belar-test.ipynb diff --git a/examples/belar-test.ipynb b/examples/belar-test.ipynb new file mode 100644 index 000000000..e467de88b --- /dev/null +++ b/examples/belar-test.ipynb @@ -0,0 +1,544 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "780354f6", + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "4fbb39b4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/shahul/belar/examples\r\n" + ] + } + ], + "source": [ + "!pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "3e39c4b3", + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(\"/home/shahul/belar\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "3e398a03", + "metadata": {}, + "outputs": [], + "source": [ + "from belar.metrics import EntailmentScore" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "9bc4b9b1", + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "0f65ccc7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:datasets.builder:Using custom data configuration explodinggradients--eli5-test-217d92ce20e19249\n", + "WARNING:datasets.builder:Found cached dataset parquet (/home/shahul/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--eli5-test-217d92ce20e19249/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n" + ] + } + ], + "source": [ + "ds = load_dataset(\"explodinggradients/eli5-test\", split=\"test_eli5\")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "fbbb3f54", + "metadata": {}, + "outputs": [], + "source": [ + "?Qsquare" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "ef5df210", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", + "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", + "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", + "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", + "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n" + ] + } + ], + "source": [ + "obj = Qsquare(\n", + " qa_model_name=\"consciousAI/question-answering-roberta-base-s\",\n", + " qg_model_name=\"mrm8488/t5-base-finetuned-question-generation-ap\",\n", + " max_answers=10,\n", + " device=\"cuda\",\n", + " save_results=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "066cc327", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Dataset({\n", + " features: ['context', 'prompt', 'ground_truth', 'references', 'generated_text'],\n", + " num_rows: 500\n", + "})" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "ba9d48fe", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:datasets.arrow_dataset:Loading cached shuffled indices for dataset at /home/shahul/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--eli5-test-217d92ce20e19249/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-191939fae858ff85.arrow\n", + "WARNING:datasets.arrow_dataset:Loading cached shuffled indices for dataset at /home/shahul/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--eli5-test-217d92ce20e19249/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-191939fae858ff85.arrow\n", + "WARNING:datasets.arrow_dataset:Loading cached shuffled indices for dataset at /home/shahul/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--eli5-test-217d92ce20e19249/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-191939fae858ff85.arrow\n" + ] + } + ], + "source": [ + "gnd_truths = [item[\"ground_truth\"][0] for item in ds.shuffle(44).select(range(0, 10))]\n", + "prompts = [item[\"prompt\"] for item in ds.shuffle(44).select(range(0, 10))]\n", + "gen_answers = [item[\"generated_text\"] for item in ds.shuffle(44).select(range(0, 10))]" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "f237c2f3", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 1e+03 ns, sys: 0 ns, total: 1e+03 ns\n", + "Wall time: 4.29 µs\n" + ] + } + ], + "source": [ + "%time\n", + "scores = obj.score(gnd_truths, gen_answers, max_length=256)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "ce35da9d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.3333333322222222,\n", + " 0.0,\n", + " 0.1999999996,\n", + " 0.1999999996,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0,\n", + " 0.4999999975,\n", + " 0.0,\n", + " 0.0]" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scores" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "74cbbb92", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "81cd7c5f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['1']" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(np.setdiff1d([\"1\", \"2\"], [\"2\"]))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "55ce2605", + "metadata": {}, + "outputs": [], + "source": [ + "dic = {\"1\": 1}" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "c481ef34", + "metadata": {}, + "outputs": [], + "source": [ + "dic.__setitem__(\"2\", 2)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "71566465", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{2, 3}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "set([2]).union(set([3]))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "84763f79", + "metadata": {}, + "outputs": [], + "source": [ + "import spacy\n", + "\n", + "nlp = spacy.load(\"en_core_web_sm\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f3c77cac", + "metadata": {}, + "outputs": [], + "source": [ + "text = \"Albert Einstein was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. Best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "20ee218e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Albert Einstein,\n", + " a German-born theoretical physicist,\n", + " the greatest and most influential physicists,\n", + " all time,\n", + " the theory,\n", + " relativity,\n", + " he,\n", + " important contributions,\n", + " the development,\n", + " the theory,\n", + " quantum mechanics]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[i for i in nlp(text).noun_chunks][0]." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "092888e8", + "metadata": {}, + "outputs": [], + "source": [ + "for i, j, k in zip([1], [2], [3]):\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "47409a31", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", + "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", + "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", + "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n" + ] + } + ], + "source": [ + "obj = EntailmentScore()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cd8b26c5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.00019535620231181383]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "obj.score(\n", + " [\"Who is TANF for? needy families\"],\n", + " [\"Who is TANF for?people who dont have enough money to buy food or pay bills\"],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b88f2ca3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.00019535620231181383]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "obj.score(\n", + " \"Who is TANF for? needy families\",\n", + " \"Who is TANF for?people who dont have enough money to buy food or pay bills\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b6510102", + "metadata": {}, + "outputs": [ + { + "ename": "RuntimeError", + "evalue": "no validator found for , see `arbitrary_types_allowed` in Config", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_64760/2871902715.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictor\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPredictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mallennlp_models\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpair_classification\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m predictor = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/snli_roberta-2020.06.09.tar.gz\",\n\u001b[1;32m 5\u001b[0m predictor_name=\"textual_entailment\")\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/predictors/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0ma\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mPredictor\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mthat\u001b[0m \u001b[0mwraps\u001b[0m \u001b[0mit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \"\"\"\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictor\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPredictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msentence_tagger\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSentenceTaggerPredictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext_classifier\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTextClassifierPredictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/predictors/predictor.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDatasetReader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mInstance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marchival\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mArchive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_archive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mutil\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/models/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marchival\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0marchive_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_archive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mArchive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic_classifier\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBasicClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultitask\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMultiTaskModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimple_tagger\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSimpleTagger\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/models/basic_classifier.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfields\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMetadataField\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mFeedForward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeq2SeqEncoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeq2VecEncoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTextFieldEmbedder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mInitializerApplicator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mutil\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_text_field_mask\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattention\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAttention\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbimpm_matching\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBiMpmMatching\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconditional_random_field\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mConditionalRandomField\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/backbones/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbone\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpretrained_transformer_backbone\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPretrainedTransformerBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvilbert_backbone\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mVilbertBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/backbones/vilbert_backbone.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocabulary\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mVocabulary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbone\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m from allennlp.modules.transformer import (\n\u001b[0m\u001b[1;32m 11\u001b[0m \u001b[0mBiModalEncoder\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mImageFeatureEmbeddings\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/transformer/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 133\u001b[0m )\n\u001b[1;32m 134\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattention_module\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSelfAttention\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mT5Attention\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 135\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactivation_layer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mActivationLayer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 136\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer_layer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAttentionLayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTransformerLayer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer_stack\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTransformerStack\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/transformer/activation_layer.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer_module\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTransformerModule\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtransformers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbert\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodeling_bert\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mACT2FN\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/transformers/models/bert/modeling_bert.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0mTokenClassifierOutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m )\n\u001b[0;32m---> 43\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m...\u001b[0m\u001b[0mmodeling_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPreTrainedModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 44\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m...\u001b[0m\u001b[0mpytorch_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mapply_chunking_to_forward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfind_pruneable_heads_and_indices\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprune_linear_layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 45\u001b[0m from ...utils import (\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 75\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_accelerate_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0maccelerate\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdispatch_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfer_auto_device_map\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minit_empty_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 77\u001b[0m from accelerate.utils import (\n\u001b[1;32m 78\u001b[0m \u001b[0mload_offloaded_weights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0m__version__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"0.15.0\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAccelerator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mbig_modeling\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcpu_offload\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisk_offload\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdispatch_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minit_empty_weights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_checkpoint_and_dispatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mlaunchers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdebug_launcher\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnotebook_launcher\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/accelerator.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mcheckpointing\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mload_accelerator_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_custom_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_accelerator_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_custom_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mdata_loader\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDataLoaderDispatcher\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprepare_data_loader\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mlogging\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_logger\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/checkpointing.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mamp\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mGradScaler\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m from .utils import (\n\u001b[0m\u001b[1;32m 25\u001b[0m \u001b[0mMODEL_NAME\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0mOPTIMIZER_NAME\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/utils/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmegatron_lm\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprepare_scheduler\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mmegatron_lm_prepare_scheduler\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmemory\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfind_executable_batch_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m from .other import (\n\u001b[0m\u001b[1;32m 123\u001b[0m \u001b[0mextract_model_from_parallel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[0mget_pretty_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/utils/other.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_deepspeed_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mdeepspeed\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDeepSpeedEngine\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_tpu_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheck_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/deepspeed/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmodule_inject\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mruntime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDeepSpeedEngine\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDeepSpeedOptimizerCallable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDeepSpeedSchedulerCallable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 18\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mruntime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mADAM_OPTIMIZER\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLAMB_OPTIMIZER\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mruntime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPipelineEngine\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/deepspeed/runtime/engine.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mdeepspeed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogging\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprint_json_dist\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mdeepspeed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minference\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDtypeEnum\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;31m# Set to torch's distributed package or deepspeed.comm based inside DeepSpeedEngine init\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/deepspeed/inference/config.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mDeepSpeedTPConfig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDeepSpeedConfigModel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 43\u001b[0m \u001b[0;34m\"\"\" Configure tensor parallelism settings \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/main.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.main.ModelMetaclass.__new__\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/fields.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.fields.ModelField.infer\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/fields.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.fields.ModelField.__init__\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/fields.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.fields.ModelField.prepare\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/fields.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.fields.ModelField.populate_validators\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/validators.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mfind_validators\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mRuntimeError\u001b[0m: no validator found for , see `arbitrary_types_allowed` in Config" + ] + } + ], + "source": [ + "from allennlp.predictors.predictor import Predictor\n", + "import allennlp_models.pair_classification\n", + "\n", + "predictor = Predictor.from_path(\n", + " \"https://storage.googleapis.com/allennlp-public-models/snli_roberta-2020.06.09.tar.gz\",\n", + " predictor_name=\"textual_entailment\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "93adc3ea", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <cell line: 2>:2                                                                              │\n",
+       "│                                                                                                  │\n",
+       "│   1 import numpy as np                                                                           │\n",
+       "│ ❱ 2 sum([[1,2],[3,4,5]])                                                                         │\n",
+       "│   3                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "TypeError: unsupported operand type(s) for +: 'int' and 'list'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[94mimport\u001b[0m \u001b[4;96mnumpy\u001b[0m \u001b[94mas\u001b[0m \u001b[4;96mnp\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2 \u001b[96msum\u001b[0m([[\u001b[94m1\u001b[0m,\u001b[94m2\u001b[0m],[\u001b[94m3\u001b[0m,\u001b[94m4\u001b[0m,\u001b[94m5\u001b[0m]]) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mTypeError: \u001b[0munsupported operand \u001b[1;35mtype\u001b[0m\u001b[1m(\u001b[0ms\u001b[1m)\u001b[0m for +: \u001b[32m'int'\u001b[0m and \u001b[32m'list'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import numpy as np\n", + "\n", + "sum([[1, 2], [3, 4, 5]])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19c25311", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "OpenAssistant", + "language": "python", + "name": "openassistant" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/quickstart.ipynb b/examples/quickstart.ipynb index 5604a503f..f726fcf23 100644 --- a/examples/quickstart.ipynb +++ b/examples/quickstart.ipynb @@ -1,14 +1,8 @@ { "cells": [ { -<<<<<<< HEAD - "cell_type": "code", - "execution_count": 1, - "id": "54b66a67", -======= "cell_type": "markdown", "id": "aeb5819b", ->>>>>>> main "metadata": {}, "source": [ "# Quickstart" @@ -36,12 +30,6 @@ }, { "cell_type": "code", -<<<<<<< HEAD - "execution_count": null, - "id": "9710719d", - "metadata": {}, - "outputs": [], -======= "execution_count": 2, "id": "0b5d4d41", "metadata": {}, @@ -67,7 +55,6 @@ "output_type": "execute_result" } ], ->>>>>>> main "source": [ "from datasets import load_dataset, concatenate_datasets\n", "\n", @@ -77,34 +64,8 @@ }, { "cell_type": "code", -<<<<<<< HEAD - "execution_count": null, - "id": "23c3f231", - "metadata": {}, - "outputs": [], - "source": [ - "ds = ds.shuffle(seed=42).select(range(500))\n", - "ds.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "81205b31", - "metadata": {}, - "outputs": [], - "source": [ - "ds.column_names" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2c5671fe", -======= "execution_count": 24, "id": "0b5abd7d", ->>>>>>> main "metadata": {}, "outputs": [], "source": [ @@ -122,10 +83,6 @@ }, { "cell_type": "code", -<<<<<<< HEAD - "execution_count": null, - "id": "207d0e48", -======= "execution_count": 28, "id": "a77c805d", "metadata": {}, @@ -156,161 +113,6 @@ "cell_type": "code", "execution_count": 29, "id": "e879f51b", ->>>>>>> main - "metadata": {}, - "outputs": [], - "source": [ -<<<<<<< HEAD - "import concurrent.futures as f\n", - "from langchain.llms import OpenAI\n", - "\n", - "llm = OpenAI()\n", - "prompt = \"\"\"\n", - "{context}\n", - "with the above context explain like I'm five: {prompt}\n", - "\"\"\"\n", - "\n", - "def get_answers(row):\n", - " qs, cs = row[\"prompt\"], row[\"context\"]\n", - " \n", - " generated_answers = []\n", - " with f.ThreadPoolExecutor(max_workers=10) as executor:\n", - " results = executor.map(llm, \n", - " [prompt.format(context=cs[i], prompt=qs[i]) for i in range(len(qs))])\n", - " for result in results:\n", - " generated_answers.append(result)\n", - " \n", - " row[\"generated_answers\"] = generated_answers\n", - " return row\n", - " \n", - "ds = ds.map(get_answers, batched=True, batch_size=10)" - ] - }, - { - "cell_type": "markdown", - "id": "5d93c658", - "metadata": {}, - "source": [ - "## Evalutate" -======= - "r = e.eval(ds[\"ground_truth\"], ds[\"generated_text\"])" ->>>>>>> main - ] - }, - { - "cell_type": "code", -<<<<<<< HEAD - "execution_count": 2, - "id": "076f2dbf", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset parquet (/home/jjmachan/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--eli5-test-217d92ce20e19249/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n" - ] - }, - { - "data": { - "text/plain": [ - "Dataset({\n", - " features: ['context', 'prompt', 'ground_truth', 'references', 'generated_text'],\n", - " num_rows: 500\n", - "})" - ] - }, - "execution_count": 2, -======= - "execution_count": 20, - "id": "f64c1915", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'rouge1_score': 0.27777314683149845, 'rouge2_score': 0.05593454553750915, 'rougeL_score': 0.16365190027294899, 'SBERT_cosine_score': 0.37552570906095206, 'edit_distance_score': 735.114, 'edit_ratio_score': 0.41482407945510713}" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "r" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "7c812dfe", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.27777314683149845" - ] - }, - "execution_count": 21, ->>>>>>> main - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ -<<<<<<< HEAD - "from datasets import load_dataset, concatenate_datasets\n", - "\n", - "ds = load_dataset(\"explodinggradients/eli5-test\", split=\"test_eli5\")\n", - "ds" -======= - "r[\"rouge1_score\"]" ->>>>>>> main - ] - }, - { - "cell_type": "code", -<<<<<<< HEAD - "execution_count": 11, - "id": "7c0cda03", - "metadata": {}, - "outputs": [], - "source": [ - "from belar.metrics import Rouge1, Evaluation, Rouge2, RougeL, SBERTScore, EntailmentScore, EditRatio, EditDistance" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "887b613c", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n" - ] - } - ], - "source": [ - "sbert_score = SBERTScore(similarity_metric=\"cosine\")\n", - "entail = EntailmentScore()\n", - "\n", - "e = Evaluation(\n", - " metrics=[Rouge1, Rouge2, RougeL, sbert_score, EditDistance, EditRatio],\n", - " batched=False, batch_size=30)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "32e338ad", "metadata": {}, "outputs": [ { @@ -335,82 +137,28 @@ { "cell_type": "code", "execution_count": 20, - "id": "b90661cb", - "metadata": {}, -======= - "execution_count": 22, - "id": "4c8c51b1", + "id": "f64c1915", "metadata": {}, ->>>>>>> main "outputs": [ { "data": { "text/plain": [ -<<<<<<< HEAD "{'rouge1_score': 0.27777314683149845, 'rouge2_score': 0.05593454553750915, 'rougeL_score': 0.16365190027294899, 'SBERT_cosine_score': 0.37552570906095206, 'edit_distance_score': 735.114, 'edit_ratio_score': 0.41482407945510713}" ] }, "execution_count": 20, -======= - "{'rouge1_score': {'mean': 0.27777314683149845,\n", - " '25%': 0.22222222222222224,\n", - " '50%': 0.28116554054054055,\n", - " '75%': 0.33333333333333337,\n", - " 'min': 0.03333333333333333,\n", - " 'max': 0.49498327759197325,\n", - " 'std': 0.07709937733409833},\n", - " 'rouge2_score': {'mean': 0.05593454553750915,\n", - " '25%': 0.029795467108899944,\n", - " '50%': 0.05203595980962454,\n", - " '75%': 0.07713675213675214,\n", - " 'min': 0.0,\n", - " 'max': 0.22499999999999998,\n", - " 'std': 0.03659179594928787},\n", - " 'rougeL_score': {'mean': 0.16365190027294899,\n", - " '25%': 0.13122438524590163,\n", - " '50%': 0.1639344262295082,\n", - " '75%': 0.19366875300914782,\n", - " 'min': 0.03333333333333333,\n", - " 'max': 0.3087248322147651,\n", - " 'std': 0.04582111082128693},\n", - " 'SBERT_cosine_score': {'mean': 0.37552570906095206,\n", - " '25%': 0.2123386301100254,\n", - " '50%': 0.33269713819026947,\n", - " '75%': 0.5326416194438934,\n", - " 'min': 0.007017173804342747,\n", - " 'max': 0.9106802940368652,\n", - " 'std': 0.2075585785391846},\n", - " 'edit_distance_score': {'mean': 735.114,\n", - " '25%': 311.5,\n", - " '50%': 476.5,\n", - " '75%': 864.25,\n", - " 'min': 106,\n", - " 'max': 6370,\n", - " 'std': 729.5287718822336},\n", - " 'edit_ratio_score': {'mean': 0.41482407945510713,\n", - " '25%': 0.39987631416202846,\n", - " '50%': 0.42918677093154384,\n", - " '75%': 0.4495093721921233,\n", - " 'min': 0.10218156228008446,\n", - " 'max': 0.5729166666666667,\n", - " 'std': 0.05807177049561045}}" - ] - }, - "execution_count": 22, ->>>>>>> main "metadata": {}, "output_type": "execute_result" } ], "source": [ -<<<<<<< HEAD "r" ] }, { "cell_type": "code", "execution_count": 21, - "id": "8c926330", + "id": "7c812dfe", "metadata": {}, "outputs": [ { @@ -425,13 +173,13 @@ } ], "source": [ - "r['rouge1_score']" + "r[\"rouge1_score\"]" ] }, { "cell_type": "code", "execution_count": 22, - "id": "d65834d4", + "id": "4c8c51b1", "metadata": {}, "outputs": [ { @@ -487,580 +235,13 @@ } ], "source": [ -======= ->>>>>>> main "r.describe()" ] }, { "cell_type": "code", "execution_count": null, -<<<<<<< HEAD - "id": "b59d1d8a", - "metadata": {}, - "outputs": [], - "source": [ - "t_not_batched = ds_eval[\"rouge1_score\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29814470", - "metadata": {}, - "outputs": [], - "source": [ - "np.array(t_batched) - np.array(t_not_batched)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "0e1da651", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['ground_truth', 'generated_text', 'SBERT_cosine_score']" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_eval.column_names" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "52bb6cee", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[0.3033774197101593,\n", - " 0.016349632292985916,\n", - " 0.4478442072868347,\n", - " 0.1860141158103943,\n", - " 0.03600190579891205,\n", - " 0.6023079752922058,\n", - " 0.289838969707489,\n", - " 0.08502114564180374,\n", - " 0.17191164195537567,\n", - " 0.3593299984931946,\n", - " 0.1715232878923416,\n", - " 0.3805505037307739,\n", - " 0.5519564151763916,\n", - " 0.2677731215953827,\n", - " 0.6183438301086426,\n", - " 0.10611602663993835,\n", - " 0.19605034589767456,\n", - " 0.08165217190980911,\n", - " 0.29304254055023193,\n", - " 0.35943326354026794,\n", - " 0.38164564967155457,\n", - " 0.03771442547440529,\n", - " 0.11554502695798874,\n", - " 0.47948333621025085,\n", - " 0.23276342451572418,\n", - " 0.4236215353012085,\n", - " 0.1943129450082779,\n", - " 0.1942053735256195,\n", - " 0.12668733298778534,\n", - " 0.2597537338733673,\n", - " 0.33301281929016113,\n", - " 0.3094521462917328,\n", - " 0.3279588520526886,\n", - " 0.32722654938697815,\n", - " 0.38284799456596375,\n", - " 0.2851578891277313,\n", - " 0.23893719911575317,\n", - " 0.6166086196899414,\n", - " 0.2423057109117508,\n", - " 0.7267876267433167,\n", - " 0.08813111484050751,\n", - " 0.48606470227241516,\n", - " 0.6568448543548584,\n", - " 0.1358499825000763,\n", - " 0.4515664577484131,\n", - " 0.23441915214061737,\n", - " 0.4741160571575165,\n", - " 0.18968994915485382,\n", - " 0.382995069026947,\n", - " 0.7173715233802795,\n", - " 0.7269276976585388,\n", - " 0.2834068834781647,\n", - " 0.2564486265182495,\n", - " 0.9106802940368652,\n", - " 0.3905271291732788,\n", - " 0.1269465684890747,\n", - " 0.09796524047851562,\n", - " 0.6954237222671509,\n", - " 0.49959367513656616,\n", - " 0.3481505811214447,\n", - " 0.2524052858352661,\n", - " 0.20396579802036285,\n", - " 0.4261414706707001,\n", - " 0.35149627923965454,\n", - " 0.060562025755643845,\n", - " 0.29626941680908203,\n", - " 0.33264321088790894,\n", - " 0.32353609800338745,\n", - " 0.0929298847913742,\n", - " 0.694779634475708,\n", - " 0.42692476511001587,\n", - " 0.6740735769271851,\n", - " 0.26791706681251526,\n", - " 0.30361559987068176,\n", - " 0.6142315864562988,\n", - " 0.8581538200378418,\n", - " 0.1934203803539276,\n", - " 0.17560303211212158,\n", - " 0.39025163650512695,\n", - " 0.2257130742073059,\n", - " 0.10104137659072876,\n", - " 0.5671371221542358,\n", - " 0.2376122921705246,\n", - " 0.7245509624481201,\n", - " 0.33550819754600525,\n", - " 0.16170960664749146,\n", - " 0.3289082944393158,\n", - " 0.21686506271362305,\n", - " 0.5573591589927673,\n", - " 0.39316579699516296,\n", - " 0.3452097177505493,\n", - " 0.7620242238044739,\n", - " 0.612403154373169,\n", - " 0.20761919021606445,\n", - " 0.3436463177204132,\n", - " 0.35804855823516846,\n", - " 0.5422661304473877,\n", - " 0.2482432872056961,\n", - " 0.24608035385608673,\n", - " 0.43996310234069824,\n", - " 0.7638659477233887,\n", - " 0.4832608997821808,\n", - " 0.3723938763141632,\n", - " 0.16313855350017548,\n", - " 0.17755097150802612,\n", - " 0.7125013470649719,\n", - " 0.21019332110881805,\n", - " 0.2878414988517761,\n", - " 0.7330911755561829,\n", - " 0.5391034483909607,\n", - " 0.3856879770755768,\n", - " 0.21089066565036774,\n", - " 0.21917514503002167,\n", - " 0.5970359444618225,\n", - " 0.10427114367485046,\n", - " 0.5017147660255432,\n", - " 0.32604700326919556,\n", - " 0.26022183895111084,\n", - " 0.2217114269733429,\n", - " 0.5664410591125488,\n", - " 0.6097017526626587,\n", - " 0.6790091395378113,\n", - " 0.6737412810325623,\n", - " 0.3198738396167755,\n", - " 0.3233138620853424,\n", - " 0.27815982699394226,\n", - " 0.5739132165908813,\n", - " 0.8073441982269287,\n", - " 0.393609881401062,\n", - " 0.34070584177970886,\n", - " 0.1426166594028473,\n", - " 0.3649061918258667,\n", - " 0.21035610139369965,\n", - " 0.15468955039978027,\n", - " 0.15301679074764252,\n", - " 0.3864727020263672,\n", - " 0.3432256877422333,\n", - " 0.27995312213897705,\n", - " 0.45306405425071716,\n", - " 0.152155339717865,\n", - " 0.5590802431106567,\n", - " 0.14337098598480225,\n", - " 0.5684935450553894,\n", - " 0.06331620365381241,\n", - " 0.7308592200279236,\n", - " 0.3433731496334076,\n", - " 0.49904948472976685,\n", - " 0.24472254514694214,\n", - " 0.17057321965694427,\n", - " 0.17359305918216705,\n", - " 0.1405472606420517,\n", - " 0.21779431402683258,\n", - " 0.6882146596908569,\n", - " 0.39259153604507446,\n", - " 0.5250310301780701,\n", - " 0.29845374822616577,\n", - " 0.6535312533378601,\n", - " 0.3323957920074463,\n", - " 0.6179606318473816,\n", - " 0.6263958215713501,\n", - " 0.25900962948799133,\n", - " 0.35419002175331116,\n", - " 0.33175551891326904,\n", - " 0.1691923886537552,\n", - " 0.6974550485610962,\n", - " 0.5213074088096619,\n", - " 0.032654277980327606,\n", - " 0.34367528557777405,\n", - " 0.405593603849411,\n", - " 0.08452585339546204,\n", - " 0.11424578726291656,\n", - " 0.6650150418281555,\n", - " 0.2742277681827545,\n", - " 0.28393787145614624,\n", - " 0.29564306139945984,\n", - " 0.5309538245201111,\n", - " 0.022119097411632538,\n", - " 0.5228688716888428,\n", - " 0.6862163543701172,\n", - " 0.4796127676963806,\n", - " 0.331642746925354,\n", - " 0.469801127910614,\n", - " 0.2787094712257385,\n", - " 0.15432526171207428,\n", - " 0.13090954720973969,\n", - " 0.5296900272369385,\n", - " 0.5006809830665588,\n", - " 0.31476107239723206,\n", - " 0.6327821612358093,\n", - " 0.27751827239990234,\n", - " 0.08453290164470673,\n", - " 0.152990460395813,\n", - " 0.2828467786312103,\n", - " 0.21192562580108643,\n", - " 0.23361067473888397,\n", - " 0.1100977212190628,\n", - " 0.729167640209198,\n", - " 0.25679513812065125,\n", - " 0.29639971256256104,\n", - " 0.19549258053302765,\n", - " 0.01892801746726036,\n", - " 0.7945613265037537,\n", - " 0.7499642372131348,\n", - " 0.15835057199001312,\n", - " 0.6000410914421082,\n", - " 0.38472887873649597,\n", - " 0.27581414580345154,\n", - " 0.6135129332542419,\n", - " 0.30333641171455383,\n", - " 0.6530413627624512,\n", - " 0.32561489939689636,\n", - " 0.6843974590301514,\n", - " 0.7383497953414917,\n", - " 0.1791287064552307,\n", - " 0.15797390043735504,\n", - " 0.1897229701280594,\n", - " 0.34278005361557007,\n", - " 0.523197591304779,\n", - " 0.2993963062763214,\n", - " 0.24305762350559235,\n", - " 0.2124125361442566,\n", - " 0.23200078308582306,\n", - " 0.5277230739593506,\n", - " 0.3923065960407257,\n", - " 0.2338612824678421,\n", - " 0.6605720520019531,\n", - " 0.4534214735031128,\n", - " 0.7204974889755249,\n", - " 0.4256589412689209,\n", - " 0.1377628594636917,\n", - " 0.1862977296113968,\n", - " 0.6173402070999146,\n", - " 0.2129381150007248,\n", - " 0.18199223279953003,\n", - " 0.4077472388744354,\n", - " 0.5461190938949585,\n", - " 0.7703336477279663,\n", - " 0.7089384198188782,\n", - " 0.12397469580173492,\n", - " 0.3445894420146942,\n", - " 0.29747506976127625,\n", - " 0.12937960028648376,\n", - " 0.6808912754058838,\n", - " 0.44350528717041016,\n", - " 0.0622265450656414,\n", - " 0.800916314125061,\n", - " 0.196528360247612,\n", - " 0.40886160731315613,\n", - " 0.5457544326782227,\n", - " 0.7547292113304138,\n", - " 0.17570790648460388,\n", - " 0.33092451095581055,\n", - " 0.3909622132778168,\n", - " 0.1750270575284958,\n", - " 0.21135497093200684,\n", - " 0.2844017744064331,\n", - " 0.6711058616638184,\n", - " 0.7111238241195679,\n", - " 0.39750146865844727,\n", - " 0.3603275716304779,\n", - " 0.20594996213912964,\n", - " 0.26992928981781006,\n", - " 0.32206788659095764,\n", - " 0.5537823438644409,\n", - " 0.6196168065071106,\n", - " 0.17448124289512634,\n", - " 0.8145052194595337,\n", - " 0.13209058344364166,\n", - " 0.6009707450866699,\n", - " 0.1729992777109146,\n", - " 0.605941891670227,\n", - " 0.16112592816352844,\n", - " 0.7443314790725708,\n", - " 0.27183473110198975,\n", - " 0.6732509732246399,\n", - " 0.34409621357917786,\n", - " 0.6225290894508362,\n", - " 0.7111546397209167,\n", - " 0.25248128175735474,\n", - " 0.25385937094688416,\n", - " 0.4553792476654053,\n", - " 0.007017173804342747,\n", - " 0.5378240942955017,\n", - " 0.6920719146728516,\n", - " 0.5118893980979919,\n", - " 0.7575575113296509,\n", - " 0.053049687296152115,\n", - " 0.34726738929748535,\n", - " 0.625588595867157,\n", - " 0.2684467136859894,\n", - " 0.21171455085277557,\n", - " 0.16874279081821442,\n", - " 0.6806609034538269,\n", - " 0.6409006118774414,\n", - " 0.43180617690086365,\n", - " 0.36487576365470886,\n", - " 0.25573742389678955,\n", - " 0.8596686124801636,\n", - " 0.7924257516860962,\n", - " 0.2288934737443924,\n", - " 0.37159034609794617,\n", - " 0.21388927102088928,\n", - " 0.7443233132362366,\n", - " 0.1677546203136444,\n", - " 0.590474009513855,\n", - " 0.2609856426715851,\n", - " 0.2530490458011627,\n", - " 0.26618924736976624,\n", - " 0.25583404302597046,\n", - " 0.20902562141418457,\n", - " 0.5943877696990967,\n", - " 0.07199332863092422,\n", - " 0.44120875000953674,\n", - " 0.3591962456703186,\n", - " 0.6544501781463623,\n", - " 0.12697549164295197,\n", - " 0.3532907962799072,\n", - " 0.4480339288711548,\n", - " 0.7042593359947205,\n", - " 0.11615218967199326,\n", - " 0.6357651948928833,\n", - " 0.24792085587978363,\n", - " 0.3313771188259125,\n", - " 0.5221624970436096,\n", - " 0.35108593106269836,\n", - " 0.135896697640419,\n", - " 0.15817011892795563,\n", - " 0.8391244411468506,\n", - " 0.2277119904756546,\n", - " 0.04543468356132507,\n", - " 0.25068429112434387,\n", - " 0.1133192926645279,\n", - " 0.28534117341041565,\n", - " 0.8111948370933533,\n", - " 0.3385901153087616,\n", - " 0.49840831756591797,\n", - " 0.4116763174533844,\n", - " 0.16915757954120636,\n", - " 0.3262860178947449,\n", - " 0.10765945911407471,\n", - " 0.1261938512325287,\n", - " 0.3500753939151764,\n", - " 0.2676033079624176,\n", - " 0.6120821833610535,\n", - " 0.62961345911026,\n", - " 0.27265217900276184,\n", - " 0.7611227035522461,\n", - " 0.2189398556947708,\n", - " 0.271114706993103,\n", - " 0.7538965940475464,\n", - " 0.1766694337129593,\n", - " 0.26010769605636597,\n", - " 0.14162400364875793,\n", - " 0.15965068340301514,\n", - " 0.30319979786872864,\n", - " 0.23467262089252472,\n", - " 0.7990760207176208,\n", - " 0.3484833538532257,\n", - " 0.3364700973033905,\n", - " 0.36943286657333374,\n", - " 0.37875810265541077,\n", - " 0.5377050042152405,\n", - " 0.2255283147096634,\n", - " 0.6214497089385986,\n", - " 0.572303295135498,\n", - " 0.5672966241836548,\n", - " 0.4602000117301941,\n", - " 0.6925125122070312,\n", - " 0.19061176478862762,\n", - " 0.750962495803833,\n", - " 0.057794470340013504,\n", - " 0.22833339869976044,\n", - " 0.12149019539356232,\n", - " 0.5187497735023499,\n", - " 0.43326133489608765,\n", - " 0.7459068298339844,\n", - " 0.28757017850875854,\n", - " 0.060881346464157104,\n", - " 0.19995999336242676,\n", - " 0.2332974374294281,\n", - " 0.5807837843894958,\n", - " 0.4985215663909912,\n", - " 0.2317824810743332,\n", - " 0.20419657230377197,\n", - " 0.2929933965206146,\n", - " 0.22726529836654663,\n", - " 0.36383742094039917,\n", - " 0.26542332768440247,\n", - " 0.33275106549263,\n", - " 0.1817902773618698,\n", - " 0.019586173817515373,\n", - " 0.6501842737197876,\n", - " 0.5130109786987305,\n", - " 0.04855664074420929,\n", - " 0.327665239572525,\n", - " 0.33484169840812683,\n", - " 0.18408897519111633,\n", - " 0.8089461326599121,\n", - " 0.2609926760196686,\n", - " 0.35048383474349976,\n", - " 0.3380715847015381,\n", - " 0.19198913872241974,\n", - " 0.47304245829582214,\n", - " 0.13059648871421814,\n", - " 0.388828307390213,\n", - " 0.6691229939460754,\n", - " 0.1510116457939148,\n", - " 0.20976220071315765,\n", - " 0.4316028952598572,\n", - " 0.5592595934867859,\n", - " 0.4931623339653015,\n", - " 0.40056753158569336,\n", - " 0.1390654295682907,\n", - " 0.7112942337989807,\n", - " 0.30744668841362,\n", - " 0.2824617028236389,\n", - " 0.29495444893836975,\n", - " 0.8129028081893921,\n", - " 0.04778153821825981,\n", - " 0.3677351772785187,\n", - " 0.38807204365730286,\n", - " 0.23143930733203888,\n", - " 0.3730814754962921,\n", - " 0.3903065323829651,\n", - " 0.10604582726955414,\n", - " 0.375832736492157,\n", - " 0.32024890184402466,\n", - " 0.3080943822860718,\n", - " 0.6008120775222778,\n", - " 0.8878772258758545,\n", - " 0.4099455773830414,\n", - " 0.4919497072696686,\n", - " 0.21881842613220215,\n", - " 0.7104718089103699,\n", - " 0.40945085883140564,\n", - " 0.7066667675971985,\n", - " 0.3884510099887848,\n", - " 0.29029491543769836,\n", - " 0.48201748728752136,\n", - " 0.645422637462616,\n", - " 0.46089968085289,\n", - " 0.26423460245132446,\n", - " 0.3575299084186554,\n", - " 0.12025940418243408,\n", - " 0.3637012839317322,\n", - " 0.5629667043685913,\n", - " 0.21808886528015137,\n", - " 0.20087826251983643,\n", - " 0.19176578521728516,\n", - " 0.521368145942688,\n", - " 0.4651867747306824,\n", - " 0.2771470844745636,\n", - " 0.15467087924480438,\n", - " 0.06321043521165848,\n", - " 0.727550208568573,\n", - " 0.6326872706413269,\n", - " 0.2524058222770691,\n", - " 0.40928635001182556,\n", - " 0.2859940230846405,\n", - " 0.24548542499542236,\n", - " 0.25654155015945435,\n", - " 0.1554943472146988,\n", - " 0.2810353636741638,\n", - " 0.39291778206825256,\n", - " 0.7448244094848633,\n", - " 0.36232057213783264,\n", - " 0.2249329537153244,\n", - " 0.5934489369392395,\n", - " 0.36474189162254333,\n", - " 0.16170084476470947,\n", - " 0.2098686695098877,\n", - " 0.3690999746322632,\n", - " 0.6965110898017883,\n", - " 0.21211691200733185,\n", - " 0.6880887150764465,\n", - " 0.7315702438354492,\n", - " 0.2110704928636551,\n", - " 0.8123224973678589,\n", - " 0.7990055680274963,\n", - " 0.14683164656162262,\n", - " 0.25454556941986084,\n", - " 0.11940312385559082,\n", - " 0.2454526573419571,\n", - " 0.5912683010101318,\n", - " 0.4947351813316345,\n", - " 0.4511561691761017,\n", - " 0.13149523735046387,\n", - " 0.1972067654132843,\n", - " 0.3593907356262207,\n", - " 0.5928145051002502,\n", - " 0.25529202818870544,\n", - " 0.2567807137966156,\n", - " 0.20362421870231628,\n", - " 0.30127424001693726,\n", - " 0.6847882270812988,\n", - " 0.6155568957328796,\n", - " 0.2527444660663605,\n", - " 0.4813864529132843,\n", - " 0.3825063407421112,\n", - " 0.3193434178829193]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_eval['SBERT_cosine_score']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50eba21d", -======= "id": "ebf0a29d", ->>>>>>> main "metadata": {}, "outputs": [], "source": [ From 36745b8cdfe6e92af9f7de6895da2c92c73df750 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 13:47:15 +0000 Subject: [PATCH 18/22] fix imports --- belar/metrics/__init__.py | 3 ++- belar/metrics/factual.py | 12 ++++++++---- tests/benchmarks/benchmark.py | 12 ++---------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/belar/metrics/__init__.py b/belar/metrics/__init__.py index e9bef643f..70fe4a3e5 100644 --- a/belar/metrics/__init__.py +++ b/belar/metrics/__init__.py @@ -1,7 +1,8 @@ from belar.metrics.base import Evaluation, Metric from belar.metrics.factual import EntailmentScore from belar.metrics.similarity import SBERTScore -from belar.metrics.simple import BLUE, EditDistance, EditRatio, Rouge1, Rouge2, RougeL +from belar.metrics.simple import (BLUE, EditDistance, EditRatio, Rouge1, + Rouge2, RougeL) __all__ = [ "Evaluation", diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 7f182063a..7df9188f1 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -1,9 +1,15 @@ from __future__ import annotations +import json +import re +import string import typing as t from dataclasses import dataclass -from transformers import AutoModelForSequenceClassification, AutoTokenizer +import numpy as np +import transformers +from transformers import (AutoConfig, AutoModelForSequenceClassification, + AutoTokenizer, PreTrainedModel) from belar.metrics import Metric from belar.utils import device_check @@ -11,11 +17,9 @@ if t.TYPE_CHECKING: from torch import device as Device - from transformers.models.auto.modeling_auto import ( - MODEL_WITH_LM_HEAD_MAPPING_NAMES, MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES, -) + MODEL_WITH_LM_HEAD_MAPPING_NAMES) MODEL_MAPPINGS_NAMES = [ MODEL_WITH_LM_HEAD_MAPPING_NAMES, diff --git a/tests/benchmarks/benchmark.py b/tests/benchmarks/benchmark.py index 12732828b..50d63beb5 100644 --- a/tests/benchmarks/benchmark.py +++ b/tests/benchmarks/benchmark.py @@ -5,16 +5,8 @@ from tqdm import tqdm from utils import print_table, timeit -from belar.metrics import ( - EditDistance, - EditRatio, - EntailmentScore, - Evaluation, - Rouge1, - Rouge2, - RougeL, - SBERTScore, -) +from belar.metrics import (EditDistance, EditRatio, EntailmentScore, + Evaluation, Rouge1, Rouge2, RougeL, SBERTScore) DEVICE = "cuda" if is_available() else "cpu" BATCHES = [0, 1] From 14075247168743116ca390e0b3c8e1a861ad78ef Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 13:53:53 +0000 Subject: [PATCH 19/22] add spacy --- belar/metrics/factual.py | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 7df9188f1..e51afd449 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -7,6 +7,7 @@ from dataclasses import dataclass import numpy as np +import spacy import transformers from transformers import (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer, PreTrainedModel) diff --git a/pyproject.toml b/pyproject.toml index e1af83e8d..2c84c3a99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "sentence-transformers", "nltk", "datasets", + "spacy", ] dynamic = ["version", "readme"] From 3b16ac86d6efb9ee919e9aec83ccf30dfb959d38 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 13:55:06 +0000 Subject: [PATCH 20/22] merge from main --- belar/metrics/__init__.py | 3 ++- belar/utils.py | 1 + tests/benchmarks/benchmark.py | 12 ++---------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/belar/metrics/__init__.py b/belar/metrics/__init__.py index e9bef643f..70fe4a3e5 100644 --- a/belar/metrics/__init__.py +++ b/belar/metrics/__init__.py @@ -1,7 +1,8 @@ from belar.metrics.base import Evaluation, Metric from belar.metrics.factual import EntailmentScore from belar.metrics.similarity import SBERTScore -from belar.metrics.simple import BLUE, EditDistance, EditRatio, Rouge1, Rouge2, RougeL +from belar.metrics.simple import (BLUE, EditDistance, EditRatio, Rouge1, + Rouge2, RougeL) __all__ = [ "Evaluation", diff --git a/belar/utils.py b/belar/utils.py index 403a1cc98..cc9eb84fa 100644 --- a/belar/utils.py +++ b/belar/utils.py @@ -1,4 +1,5 @@ from __future__ import annotations + import typing as t from warnings import warn diff --git a/tests/benchmarks/benchmark.py b/tests/benchmarks/benchmark.py index 12732828b..50d63beb5 100644 --- a/tests/benchmarks/benchmark.py +++ b/tests/benchmarks/benchmark.py @@ -5,16 +5,8 @@ from tqdm import tqdm from utils import print_table, timeit -from belar.metrics import ( - EditDistance, - EditRatio, - EntailmentScore, - Evaluation, - Rouge1, - Rouge2, - RougeL, - SBERTScore, -) +from belar.metrics import (EditDistance, EditRatio, EntailmentScore, + Evaluation, Rouge1, Rouge2, RougeL, SBERTScore) DEVICE = "cuda" if is_available() else "cpu" BATCHES = [0, 1] From ce80bbb0e45e4e828f1401b861a84e81d79d7f33 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 14:30:13 +0000 Subject: [PATCH 21/22] fix type checks --- belar/metrics/factual.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/belar/metrics/factual.py b/belar/metrics/factual.py index 9449448e7..3a9769280 100644 --- a/belar/metrics/factual.py +++ b/belar/metrics/factual.py @@ -27,7 +27,7 @@ MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES, ] - +DEVICES = ["cpu", "cuda"] SPACY_MODEL = "en_core_web_sm" LABEL2SCORE = {"entailment": 1, "contradiction": 0, "neutral": 0.5} EPS = 1e-8 @@ -125,11 +125,16 @@ def score( class QAGQ: - def __init__(self, model: PreTrainedModel, model_name_or_path: str, device="cpu"): + def __init__( + self, + model: PreTrainedModel, + model_name_or_path: str, + device: t.Literal["cpu", "cuda"] | Device = "cpu", + ): self.model = model.from_pretrained(model_name_or_path) - self.model.eval() + self.model.eval() # type: ignore self.device = device_check(device) - self.model.to(self.device) + self.model.to(self.device) # type: ignore self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) @classmethod @@ -157,7 +162,7 @@ def batch_generate_question(self, answers: list[str], context: str, **kwargs): return_tensors="pt", ) encodings = {k: v.to(self.device) for k, v in encodings.items()} - outputs = self.model.generate(**encodings, **kwargs) + outputs = self.model.generate(**encodings, **kwargs) # type: ignore outputs = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) return [output.replace("question:", "").strip() for output in outputs] @@ -174,7 +179,9 @@ def batch_generate_answers(self, questions: list[str], context: str, **kwargs): encodings = { k: v.view(-1, max_length).to(self.device) for k, v in encodings.items() } - poss_ans_starts, poss_ans_ends = self.model(**encodings, return_dict=False) + poss_ans_starts, poss_ans_ends = self.model( + **encodings, return_dict=False + ) # type: ignore best_start = poss_ans_starts.argmax(1) best_ends = poss_ans_ends.argmax(1) answers = [ From 4ed701e7fea254667837876486f09dffbd3cd2f9 Mon Sep 17 00:00:00 2001 From: shahules786 Date: Sun, 14 May 2023 14:42:11 +0000 Subject: [PATCH 22/22] rmv test notebook --- examples/belar-test.ipynb | 544 -------------------------------------- 1 file changed, 544 deletions(-) delete mode 100644 examples/belar-test.ipynb diff --git a/examples/belar-test.ipynb b/examples/belar-test.ipynb deleted file mode 100644 index e467de88b..000000000 --- a/examples/belar-test.ipynb +++ /dev/null @@ -1,544 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "780354f6", - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "4fbb39b4", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/home/shahul/belar/examples\r\n" - ] - } - ], - "source": [ - "!pwd" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "3e39c4b3", - "metadata": {}, - "outputs": [], - "source": [ - "os.chdir(\"/home/shahul/belar\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "3e398a03", - "metadata": {}, - "outputs": [], - "source": [ - "from belar.metrics import EntailmentScore" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "9bc4b9b1", - "metadata": {}, - "outputs": [], - "source": [ - "from datasets import load_dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "0f65ccc7", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:datasets.builder:Using custom data configuration explodinggradients--eli5-test-217d92ce20e19249\n", - "WARNING:datasets.builder:Found cached dataset parquet (/home/shahul/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--eli5-test-217d92ce20e19249/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n" - ] - } - ], - "source": [ - "ds = load_dataset(\"explodinggradients/eli5-test\", split=\"test_eli5\")" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "fbbb3f54", - "metadata": {}, - "outputs": [], - "source": [ - "?Qsquare" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "ef5df210", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n" - ] - } - ], - "source": [ - "obj = Qsquare(\n", - " qa_model_name=\"consciousAI/question-answering-roberta-base-s\",\n", - " qg_model_name=\"mrm8488/t5-base-finetuned-question-generation-ap\",\n", - " max_answers=10,\n", - " device=\"cuda\",\n", - " save_results=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "066cc327", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Dataset({\n", - " features: ['context', 'prompt', 'ground_truth', 'references', 'generated_text'],\n", - " num_rows: 500\n", - "})" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "ba9d48fe", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached shuffled indices for dataset at /home/shahul/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--eli5-test-217d92ce20e19249/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-191939fae858ff85.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached shuffled indices for dataset at /home/shahul/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--eli5-test-217d92ce20e19249/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-191939fae858ff85.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached shuffled indices for dataset at /home/shahul/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--eli5-test-217d92ce20e19249/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec/cache-191939fae858ff85.arrow\n" - ] - } - ], - "source": [ - "gnd_truths = [item[\"ground_truth\"][0] for item in ds.shuffle(44).select(range(0, 10))]\n", - "prompts = [item[\"prompt\"] for item in ds.shuffle(44).select(range(0, 10))]\n", - "gen_answers = [item[\"generated_text\"] for item in ds.shuffle(44).select(range(0, 10))]" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "f237c2f3", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1e+03 ns, sys: 0 ns, total: 1e+03 ns\n", - "Wall time: 4.29 µs\n" - ] - } - ], - "source": [ - "%time\n", - "scores = obj.score(gnd_truths, gen_answers, max_length=256)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "ce35da9d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[0.3333333322222222,\n", - " 0.0,\n", - " 0.1999999996,\n", - " 0.1999999996,\n", - " 0.0,\n", - " 0.0,\n", - " 0.0,\n", - " 0.4999999975,\n", - " 0.0,\n", - " 0.0]" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "scores" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "74cbbb92", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "81cd7c5f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['1']" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list(np.setdiff1d([\"1\", \"2\"], [\"2\"]))" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "55ce2605", - "metadata": {}, - "outputs": [], - "source": [ - "dic = {\"1\": 1}" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "c481ef34", - "metadata": {}, - "outputs": [], - "source": [ - "dic.__setitem__(\"2\", 2)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "71566465", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{2, 3}" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "set([2]).union(set([3]))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "84763f79", - "metadata": {}, - "outputs": [], - "source": [ - "import spacy\n", - "\n", - "nlp = spacy.load(\"en_core_web_sm\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "f3c77cac", - "metadata": {}, - "outputs": [], - "source": [ - "text = \"Albert Einstein was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. Best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics.\"" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "20ee218e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Albert Einstein,\n", - " a German-born theoretical physicist,\n", - " the greatest and most influential physicists,\n", - " all time,\n", - " the theory,\n", - " relativity,\n", - " he,\n", - " important contributions,\n", - " the development,\n", - " the theory,\n", - " quantum mechanics]" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "[i for i in nlp(text).noun_chunks][0]." - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "092888e8", - "metadata": {}, - "outputs": [], - "source": [ - "for i, j, k in zip([1], [2], [3]):\n", - " pass" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "47409a31", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n", - "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n" - ] - } - ], - "source": [ - "obj = EntailmentScore()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "cd8b26c5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[0.00019535620231181383]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "obj.score(\n", - " [\"Who is TANF for? needy families\"],\n", - " [\"Who is TANF for?people who dont have enough money to buy food or pay bills\"],\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "b88f2ca3", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[0.00019535620231181383]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "obj.score(\n", - " \"Who is TANF for? needy families\",\n", - " \"Who is TANF for?people who dont have enough money to buy food or pay bills\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "b6510102", - "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "no validator found for , see `arbitrary_types_allowed` in Config", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_64760/2871902715.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictor\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPredictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mallennlp_models\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpair_classification\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m predictor = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/snli_roberta-2020.06.09.tar.gz\",\n\u001b[1;32m 5\u001b[0m predictor_name=\"textual_entailment\")\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/predictors/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0ma\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mPredictor\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mthat\u001b[0m \u001b[0mwraps\u001b[0m \u001b[0mit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \"\"\"\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictor\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPredictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msentence_tagger\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSentenceTaggerPredictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext_classifier\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTextClassifierPredictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/predictors/predictor.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDatasetReader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mInstance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marchival\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mArchive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_archive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mutil\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/models/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marchival\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0marchive_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_archive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mArchive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic_classifier\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBasicClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultitask\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMultiTaskModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimple_tagger\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSimpleTagger\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/models/basic_classifier.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfields\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMetadataField\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mFeedForward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeq2SeqEncoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeq2VecEncoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTextFieldEmbedder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mInitializerApplicator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mutil\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_text_field_mask\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattention\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAttention\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbimpm_matching\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBiMpmMatching\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconditional_random_field\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mConditionalRandomField\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/backbones/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbone\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpretrained_transformer_backbone\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPretrainedTransformerBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvilbert_backbone\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mVilbertBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/backbones/vilbert_backbone.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocabulary\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mVocabulary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbones\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbone\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m from allennlp.modules.transformer import (\n\u001b[0m\u001b[1;32m 11\u001b[0m \u001b[0mBiModalEncoder\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mImageFeatureEmbeddings\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/transformer/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 133\u001b[0m )\n\u001b[1;32m 134\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattention_module\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSelfAttention\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mT5Attention\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 135\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactivation_layer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mActivationLayer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 136\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer_layer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAttentionLayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTransformerLayer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer_stack\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTransformerStack\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/allennlp/modules/transformer/activation_layer.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mallennlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer_module\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTransformerModule\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtransformers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbert\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodeling_bert\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mACT2FN\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/transformers/models/bert/modeling_bert.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0mTokenClassifierOutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m )\n\u001b[0;32m---> 43\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m...\u001b[0m\u001b[0mmodeling_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPreTrainedModel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 44\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m...\u001b[0m\u001b[0mpytorch_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mapply_chunking_to_forward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfind_pruneable_heads_and_indices\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprune_linear_layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 45\u001b[0m from ...utils import (\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 75\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_accelerate_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0maccelerate\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdispatch_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfer_auto_device_map\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minit_empty_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 77\u001b[0m from accelerate.utils import (\n\u001b[1;32m 78\u001b[0m \u001b[0mload_offloaded_weights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0m__version__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"0.15.0\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAccelerator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mbig_modeling\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcpu_offload\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisk_offload\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdispatch_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minit_empty_weights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_checkpoint_and_dispatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mlaunchers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdebug_launcher\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnotebook_launcher\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/accelerator.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mcheckpointing\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mload_accelerator_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_custom_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_accelerator_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_custom_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mdata_loader\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDataLoaderDispatcher\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprepare_data_loader\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mlogging\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_logger\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/checkpointing.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mamp\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mGradScaler\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m from .utils import (\n\u001b[0m\u001b[1;32m 25\u001b[0m \u001b[0mMODEL_NAME\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0mOPTIMIZER_NAME\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/utils/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmegatron_lm\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprepare_scheduler\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mmegatron_lm_prepare_scheduler\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmemory\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfind_executable_batch_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m from .other import (\n\u001b[0m\u001b[1;32m 123\u001b[0m \u001b[0mextract_model_from_parallel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[0mget_pretty_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/accelerate/utils/other.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_deepspeed_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mdeepspeed\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDeepSpeedEngine\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_tpu_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheck_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/deepspeed/__init__.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmodule_inject\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mruntime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDeepSpeedEngine\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDeepSpeedOptimizerCallable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDeepSpeedSchedulerCallable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 18\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mruntime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mADAM_OPTIMIZER\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLAMB_OPTIMIZER\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mruntime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPipelineEngine\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/deepspeed/runtime/engine.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mdeepspeed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogging\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprint_json_dist\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mdeepspeed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minference\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDtypeEnum\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;31m# Set to torch's distributed package or deepspeed.comm based inside DeepSpeedEngine init\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/deepspeed/inference/config.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mDeepSpeedTPConfig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDeepSpeedConfigModel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 43\u001b[0m \u001b[0;34m\"\"\" Configure tensor parallelism settings \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/main.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.main.ModelMetaclass.__new__\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/fields.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.fields.ModelField.infer\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/fields.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.fields.ModelField.__init__\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/fields.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.fields.ModelField.prepare\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/fields.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mpydantic.fields.ModelField.populate_validators\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/pydantic/validators.cpython-39-x86_64-linux-gnu.so\u001b[0m in \u001b[0;36mfind_validators\u001b[0;34m()\u001b[0m\n", - "\u001b[0;31mRuntimeError\u001b[0m: no validator found for , see `arbitrary_types_allowed` in Config" - ] - } - ], - "source": [ - "from allennlp.predictors.predictor import Predictor\n", - "import allennlp_models.pair_classification\n", - "\n", - "predictor = Predictor.from_path(\n", - " \"https://storage.googleapis.com/allennlp-public-models/snli_roberta-2020.06.09.tar.gz\",\n", - " predictor_name=\"textual_entailment\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "93adc3ea", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
-       "│ in <cell line: 2>:2                                                                              │\n",
-       "│                                                                                                  │\n",
-       "│   1 import numpy as np                                                                           │\n",
-       "│ ❱ 2 sum([[1,2],[3,4,5]])                                                                         │\n",
-       "│   3                                                                                              │\n",
-       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
-       "TypeError: unsupported operand type(s) for +: 'int' and 'list'\n",
-       "
\n" - ], - "text/plain": [ - "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[94mimport\u001b[0m \u001b[4;96mnumpy\u001b[0m \u001b[94mas\u001b[0m \u001b[4;96mnp\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2 \u001b[96msum\u001b[0m([[\u001b[94m1\u001b[0m,\u001b[94m2\u001b[0m],[\u001b[94m3\u001b[0m,\u001b[94m4\u001b[0m,\u001b[94m5\u001b[0m]]) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", - "\u001b[1;91mTypeError: \u001b[0munsupported operand \u001b[1;35mtype\u001b[0m\u001b[1m(\u001b[0ms\u001b[1m)\u001b[0m for +: \u001b[32m'int'\u001b[0m and \u001b[32m'list'\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import numpy as np\n", - "\n", - "sum([[1, 2], [3, 4, 5]])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19c25311", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "OpenAssistant", - "language": "python", - "name": "openassistant" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.8" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}