From 533bb59d457be074f209576a04d40661e606886c Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 8 Sep 2020 17:05:15 +0000 Subject: [PATCH] Code compatible with transformers==3.1.0. Freezing some libraries in requirements.txt, ready for Release 0.2 --- coverage.py | 2 -- model_generator.py | 18 +++++++++--------- requirements.txt | 7 +++---- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/coverage.py b/coverage.py index 2762923..f5ba1a3 100644 --- a/coverage.py +++ b/coverage.py @@ -12,7 +12,6 @@ class KeywordExtractor(): def __init__(self, n_kws=15): self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') - self.tokenizer.max_len = 10000 self.n_kws = n_kws self.bert_w2i = {w: i for i, w in enumerate(self.tokenizer.vocab)} @@ -70,7 +69,6 @@ def extract_keywords(self, unmasked): class KeywordCoverage(): def __init__(self, device, keyword_model_file, model_file=None, n_kws=15): self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') - self.tokenizer.max_len = 10000 self.vocab_size = self.tokenizer.vocab_size self.n_kws = n_kws diff --git a/model_generator.py b/model_generator.py index a87bd3c..b25fb55 100644 --- a/model_generator.py +++ b/model_generator.py @@ -73,8 +73,8 @@ def train_batch(self, bodies, summaries, special_append=None, no_preinput=False) inputs, summ_inp, summ_out = self.preprocess_batch(bodies, summaries, special_append) past = None if not no_preinput: - _, past = self.model(input_ids=inputs, past=None) - logits, _ = self.model(input_ids=summ_inp, past=past) + _, past = self.model(input_ids=inputs, past_key_values=None) + logits, _ = self.model(input_ids=summ_inp, past_key_values=past) crit = torch.nn.CrossEntropyLoss(ignore_index=-1) loss = crit(logits.view(-1, self.tokenizer.vocab_size), summ_out.contiguous().view(-1)) return loss @@ -97,11 +97,11 @@ def decode_batch(self, bodies, special_append=None, max_output_length=100, sampl # Sometimes, we process the same input, as we run it once as a sampled, and once as an argmax, in which case we should reuse the computation if input_past is None: inputs = self.preprocess_input(bodies, special_append) - _, input_past = self.model(input_ids=inputs, past=None) + _, input_past = self.model(input_ids=inputs, past_key_values=None) past = input_past while build_up is None or (build_up.shape[1] < max_output_length and not all([self.tokenizer.end_id in build for build in build_up])): - logits, past = self.model(input_ids=current, past=past) + logits, past = self.model(input_ids=current, past_key_values=past) probs = torch.nn.functional.softmax(logits, dim=2).squeeze(1) logprobs = torch.nn.functional.log_softmax(logits, dim=2) if sample: @@ -149,12 +149,12 @@ def decode_beam_batch(self, bodies, beam_size=3, max_output_length=100, sample=F one_every_k = torch.FloatTensor([1] + [0] * (beam_size-1)).repeat(batch_size*beam_size).to(self.device) # Sometimes, we process the same input, as we run it once as a sampled, and once as an argmax, in which case we should reuse the computation - _, input_past = self.model(input_ids=inputs, past=None) + _, input_past = self.model(input_ids=inputs, past_key_values=None) input_past = [torch.repeat_interleave(p, repeats=beam_size, dim=1) for p in input_past] past = input_past while build_up is None or (build_up.shape[1] < max_output_length and not all([self.tokenizer.end_id in build for build in build_up])): - logits, past = self.model(input_ids=next_words, past=past) + logits, past = self.model(input_ids=next_words, past_key_values=past) probs = torch.nn.functional.softmax(logits, dim=2).squeeze(1) logprobs = torch.nn.functional.log_softmax(logits, dim=2) @@ -254,7 +254,7 @@ def score(self, summaries, bodies, bodies_tokenized=None, lengths=None, extra=No summ_out = summ_out.contiguous() with torch.no_grad(): - logits, _ = self.model(input_ids=summ_inp, past=None) + logits, _ = self.model(input_ids=summ_inp, past_key_values=None) crit = torch.nn.CrossEntropyLoss(ignore_index=-1, reduction='none') loss = crit(logits.view(-1, self.tokenizer.vocab_size), summ_out.view(-1)).view(summ_out.shape) @@ -272,8 +272,8 @@ def score_pairs(self, bodies, summaries): inputs, summ_inp, summ_out = self.preprocess_batch(bodies, summaries) with torch.no_grad(): - _, past = self.model(input_ids=inputs, past=None) - logits, _ = self.model(input_ids=summ_inp, past=past) + _, past = self.model(input_ids=inputs, past_key_values=None) + logits, _ = self.model(input_ids=summ_inp, past_key_values=past) crit = torch.nn.CrossEntropyLoss(ignore_index=-1, reduction='none') loss = crit(logits.view(-1, self.tokenizer.vocab_size), summ_out.view(-1)).view(summ_out.shape) diff --git a/requirements.txt b/requirements.txt index 0c1dadd..82181b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ -transformers==3.0.2 -sklearn -nltk +transformers==3.1.0 +sklearn==0.22.1 +nltk==3.5 h5py tqdm matplotlib -sklearn