Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix transformers #3578

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Fix transformers #3578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
00b017c
Update is_transformers_lm
costrau Mar 12, 2024
6eedb01
Merge branch 'master' into fix-transformers
costrau Mar 15, 2024
a42c3ce
Fix formatting
costrau Mar 18, 2024
ffa2272
Remove usage of manual list MODELS_FOR_CAUSAL_LM in _topk_lm.py, use …
costrau Mar 18, 2024
aa87aa8
Remove unused lists in transformers.py
costrau Mar 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
54 changes: 28 additions & 26 deletions shap/models/_topk_lm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from .._serializable import Deserializer, Serializer
from ..utils import safe_isinstance
from ..utils.transformers import MODELS_FOR_CAUSAL_LM, getattr_silent
from ..utils.transformers import getattr_silent
from ._model import Model


Expand Down Expand Up @@ -210,31 +210,33 @@
Logits corresponding to next word/masked word.

"""
if safe_isinstance(self.inner_model, MODELS_FOR_CAUSAL_LM):
inputs = self.get_inputs(X, padding_side="left")
if self.model_type == "pt":
import torch
inputs["position_ids"] = (inputs["attention_mask"].long().cumsum(-1) - 1)
inputs["position_ids"].masked_fill_(inputs["attention_mask"] == 0, 0)
inputs = inputs.to(self.device)
# generate outputs and logits
with torch.no_grad():
outputs = self.inner_model(**inputs, return_dict=True)
# extract only logits corresponding to target sentence ids
logits = outputs.logits.detach().cpu().numpy().astype('float64')[:, -1, :]
elif self.model_type == "tf":
import tensorflow as tf
inputs["position_ids"] = tf.math.cumsum(inputs["attention_mask"], axis=-1) - 1
inputs["position_ids"] = tf.where(inputs["attention_mask"] == 0, 0, inputs["position_ids"])
if self.device is None:
outputs = self.inner_model(inputs, return_dict=True)
else:
try:
with tf.device(self.device):
outputs = self.inner_model(inputs, return_dict=True)
except RuntimeError as err:
print(err)
logits = outputs.logits.numpy().astype('float64')[:, -1, :]
if self.model_type in ["pt", "tf"]:
from transformers import MODEL_FOR_CAUSAL_LM_MAPPING
if type(self.inner_model) in MODEL_FOR_CAUSAL_LM_MAPPING.values():
inputs = self.get_inputs(X, padding_side="left")
if self.model_type == "pt":
import torch
inputs["position_ids"] = (inputs["attention_mask"].long().cumsum(-1) - 1)
inputs["position_ids"].masked_fill_(inputs["attention_mask"] == 0, 0)
inputs = inputs.to(self.device)

Check warning on line 221 in shap/models/_topk_lm.py

View check run for this annotation

Codecov / codecov/patch

shap/models/_topk_lm.py#L213-L221 

Added lines #L213 - L221 were not covered by tests
# generate outputs and logits
with torch.no_grad():
outputs = self.inner_model(**inputs, return_dict=True)

Check warning on line 224 in shap/models/_topk_lm.py

View check run for this annotation

Codecov / codecov/patch

shap/models/_topk_lm.py#L223-L224 

Added lines #L223 - L224 were not covered by tests
# extract only logits corresponding to target sentence ids
logits = outputs.logits.detach().cpu().numpy().astype('float64')[:, -1, :]
elif self.model_type == "tf":
import tensorflow as tf
inputs["position_ids"] = tf.math.cumsum(inputs["attention_mask"], axis=-1) - 1
inputs["position_ids"] = tf.where(inputs["attention_mask"] == 0, 0, inputs["position_ids"])
if self.device is None:
outputs = self.inner_model(inputs, return_dict=True)

Check warning on line 232 in shap/models/_topk_lm.py

View check run for this annotation

Codecov / codecov/patch

shap/models/_topk_lm.py#L226-L232 

Added lines #L226 - L232 were not covered by tests
else:
try:
with tf.device(self.device):
outputs = self.inner_model(inputs, return_dict=True)
except RuntimeError as err:
print(err)
logits = outputs.logits.numpy().astype('float64')[:, -1, :]

Check warning on line 239 in shap/models/_topk_lm.py

View check run for this annotation

Codecov / codecov/patch

shap/models/_topk_lm.py#L234-L239 

Added lines #L234 - L239 were not covered by tests
return logits

def save(self, out_file):
Expand Down
87 changes: 10 additions & 77 deletions shap/utils/transformers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,80 +1,5 @@
from ._general import safe_isinstance

MODELS_FOR_SEQ_TO_SEQ_CAUSAL_LM = [
"transformers.T5ForConditionalGeneration",
"transformers.PegasusForConditionalGeneration",
"transformers.MarianMTModel",
"transformers.MBartForConditionalGeneration",
"transformers.BlenderbotForConditionalGeneration",
"transformers.BartForConditionalGeneration",
"transformers.FSMTForConditionalGeneration",
"transformers.EncoderDecoderModel",
"transformers.XLMProphetNetForConditionalGeneration",
"transformers.ProphetNetForConditionalGeneration",
"transformers.TFMT5ForConditionalGeneration",
"transformers.TFT5ForConditionalGeneration",
"transformers.TFMarianMTModel",
"transformers.TFMBartForConditionalGeneration",
"transformers.TFPegasusForConditionalGeneration",
"transformers.TFBlenderbotForConditionalGeneration",
"transformers.TFBartForConditionalGeneration"
]

MODELS_FOR_CAUSAL_LM = [
"transformers.CamembertForCausalLM",
"transformers.XLMRobertaForCausalLM",
"transformers.RobertaForCausalLM",
"transformers.BertLMHeadModel",
"transformers.OpenAIGPTLMHeadModel",
"transformers.GPT2LMHeadModel",
"transformers.TransfoXLLMHeadModel",
"transformers.XLNetLMHeadModel",
"transformers.XLMWithLMHeadModel",
"transformers.CTRLLMHeadModel",
"transformers.ReformerModelWithLMHead",
"transformers.BertGenerationDecoder",
"transformers.XLMProphetNetForCausalLM",
"transformers.ProphetNetForCausalLM",
"transformers.TFBertLMHeadModel",
"transformers.TFOpenAIGPTLMHeadModel",
"transformers.TFGPT2LMHeadModel",
"transformers.TFTransfoXLLMHeadModel",
"transformers.TFXLNetLMHeadModel",
"transformers.TFXLMWithLMHeadModel",
"transformers.TFCTRLLMHeadModel",
]

MODELS_FOR_MASKED_LM = [
"transformers.LayoutLMForMaskedLM",
"transformers.DistilBertForMaskedLM",
"transformers.AlbertForMaskedLM",
"transformers.BartForConditionalGeneration",
"transformers.CamembertForMaskedLM",
"transformers.XLMRobertaForMaskedLM",
"transformers.LongformerForMaskedLM",
"transformers.RobertaForMaskedLM",
"transformers.SqueezeBertForMaskedLM",
"transformers.BertForMaskedLM",
"transformers.MobileBertForMaskedLM",
"transformers.FlaubertWithLMHeadModel",
"transformers.XLMWithLMHeadModel",
"transformers.ElectraForMaskedLM",
"transformers.ReformerForMaskedLM",
"transformers.FunnelForMaskedLM",
"transformers.TFDistilBertForMaskedLM",
"transformers.TFAlbertForMaskedLM",
"transformers.TFCamembertForMaskedLM",
"transformers.TFXLMRobertaForMaskedLM",
"transformers.TFLongformerForMaskedLM",
"transformers.TFRobertaForMaskedLM",
"transformers.TFBertForMaskedLM",
"transformers.TFMobileBertForMaskedLM",
"transformers.TFFlaubertWithLMHeadModel",
"transformers.TFXLMWithLMHeadModel",
"transformers.TFElectraForMaskedLM",
"transformers.TFFunnelForMaskedLM"
]

SENTENCEPIECE_TOKENIZERS = [
"transformers.MarianTokenizer",
"transformers.T5Tokenizer",
Expand All @@ -84,8 +9,16 @@

def is_transformers_lm(model):
"""Check if the given model object is a huggingface transformers language model."""
return (safe_isinstance(model, "transformers.PreTrainedModel") or safe_isinstance(model, "transformers.TFPreTrainedModel")) and \
safe_isinstance(model, MODELS_FOR_SEQ_TO_SEQ_CAUSAL_LM + MODELS_FOR_CAUSAL_LM)
if safe_isinstance(
model, "transformers.PreTrainedModel") or safe_isinstance(
model, "transformers.TFPreTrainedModel"):
from transformers import (
MODEL_FOR_CAUSAL_LM_MAPPING,
MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
)
return type(model) in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.values() or type(
model) in MODEL_FOR_CAUSAL_LM_MAPPING.values()
return False

def parse_prefix_suffix_for_tokenizer(tokenizer):
"""Set prefix and suffix tokens based on null tokens.
Expand Down