Skip to content

Commit

Permalink
[#41] add converstion helper method to be compliant with huggingface api
Browse files Browse the repository at this point in the history
  • Loading branch information
raymondng76 committed Dec 13, 2021
1 parent a519024 commit bcbb06d
Showing 1 changed file with 6 additions and 3 deletions.
9 changes: 6 additions & 3 deletions sgnlp/models/sentic_asgcn/tokenization.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,12 @@ def vocab_size(self):
def get_vocab(self):
return dict(self.vocab)

def _convert_token_to_id(self, token: str) -> int:
return self.vocab.get(token, self.vocab.get(self.unk_token))

def _convert_id_to_token(self, index: int) -> str:
return self.ids_to_tokens(index, self.unk_token)

@staticmethod
def __read_text_file(file_names: List[str]) -> str:
"""
Expand Down Expand Up @@ -91,6 +97,3 @@ def _tokenize(self, text, **kwargs):
if len(sequence) == 0:
sequence = [0]
return sequence

def _convert_id_to_token(self, index: int) -> str:
return super()._convert_id_to_token(index)

0 comments on commit bcbb06d

Please sign in to comment.