From 1c542df7e554a2014051dd09becf60f157fed524 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 6 Nov 2019 16:26:31 -0500 Subject: [PATCH] Add RoBERTa-based GPT-2 Output Detector from OpenAI converted from https://github.com/openai/gpt-2-output-dataset/tree/master/detector Co-Authored-By: Lysandre Debut Co-Authored-By: Jong Wook Kim Co-Authored-By: Jeff Wu --- docs/source/pretrained_models.rst | 8 ++++++++ transformers/configuration_roberta.py | 2 ++ transformers/modeling_roberta.py | 2 ++ transformers/tokenization_roberta.py | 6 ++++++ 4 files changed, 18 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 559c81cbb05ad..edb47e7f1cc62 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -127,6 +127,14 @@ Here is the full list of the currently provided pretrained models together with | | ``roberta-large-mnli`` | | 24-layer, 1024-hidden, 16-heads, 355M parameters | | | | | ``roberta-large`` fine-tuned on `MNLI `__. | | | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``roberta-base-openai-detector`` | | 12-layer, 768-hidden, 12-heads, 125M parameters | +| | | | ``roberta-base`` fine-tuned by OpenAI on the outputs of the 1.5B-parameter GPT-2 model. | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``roberta-large-openai-detector`` | | 24-layer, 1024-hidden, 16-heads, 355M parameters | +| | | | ``roberta-large`` fine-tuned by OpenAI on the outputs of the 1.5B-parameter GPT-2 model. | +| | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | DistilBERT | ``distilbert-base-uncased`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | | | | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint | diff --git a/transformers/configuration_roberta.py b/transformers/configuration_roberta.py index 367a85211d5bb..842edac56e67e 100644 --- a/transformers/configuration_roberta.py +++ b/transformers/configuration_roberta.py @@ -29,6 +29,8 @@ 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-config.json", 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-config.json", 'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-config.json", + 'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-openai-detector-config.json", + 'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-openai-detector-config.json", } diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py index 58b86000bbde3..fc27353d374ad 100644 --- a/transformers/modeling_roberta.py +++ b/transformers/modeling_roberta.py @@ -35,6 +35,8 @@ 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-pytorch_model.bin", 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-pytorch_model.bin", 'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-pytorch_model.bin", + 'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-openai-detector-pytorch_model.bin", + 'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-openai-detector-pytorch_model.bin", } class RobertaEmbeddings(BertEmbeddings): diff --git a/transformers/tokenization_roberta.py b/transformers/tokenization_roberta.py index 5e1300fa4d742..df3e12bc7c328 100644 --- a/transformers/tokenization_roberta.py +++ b/transformers/tokenization_roberta.py @@ -47,6 +47,8 @@ def lru_cache(): 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.json", 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-vocab.json", 'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-vocab.json", + 'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json", + 'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-vocab.json", }, 'merges_file': { @@ -54,6 +56,8 @@ def lru_cache(): 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-merges.txt", 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-merges.txt", 'distilroberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-merges.txt", + 'roberta-base-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt", + 'roberta-large-openai-detector': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-merges.txt", }, } @@ -62,6 +66,8 @@ def lru_cache(): 'roberta-large': 512, 'roberta-large-mnli': 512, 'distilroberta-base': 512, + 'roberta-base-openai-detector': 512, + 'roberta-large-openai-detector': 512, }