Part2(Hugging face).ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Part2(Hugging face).ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyPyF+Doer/+twTxE8Pw/g8g"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# Download and unzip the given two datasets."],"metadata":{"id":"7HZPIbKY1ryr"}},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bxMATRqmu3ae","executionInfo":{"status":"ok","timestamp":1647351131792,"user_tz":-210,"elapsed":699,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}},"outputId":"2419acba-d3b8-4f62-8873-d655eabc5f85"},"outputs":[{"output_type":"stream","name":"stdout","text":["--2022-03-15 13:32:11--  http://www.gutenberg.org/cache/epub/16457/pg16457.txt\n","Resolving www.gutenberg.org (www.gutenberg.org)... 152.19.134.47, 2610:28:3090:3000:0:bad:cafe:47\n","Connecting to www.gutenberg.org (www.gutenberg.org)|152.19.134.47|:80... connected.\n","HTTP request sent, awaiting response... 302 Found\n","Location: https://www.gutenberg.org/cache/epub/16457/pg16457.txt [following]\n","--2022-03-15 13:32:11--  https://www.gutenberg.org/cache/epub/16457/pg16457.txt\n","Connecting to www.gutenberg.org (www.gutenberg.org)|152.19.134.47|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 617622 (603K) [text/plain]\n","Saving to: ‘pg16457.txt’\n","\n","pg16457.txt         100%[===================>] 603.15K  --.-KB/s    in 0.1s    \n","\n","2022-03-15 13:32:11 (4.85 MB/s) - ‘pg16457.txt’ saved [617622/617622]\n","\n"]}],"source":["!wget http://www.gutenberg.org/cache/epub/16457/pg16457.txt"]},{"cell_type":"code","source":["!wget https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-raw-v1.zip"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5hmxs2cRyr0N","executionInfo":{"status":"ok","timestamp":1647351135560,"user_tz":-210,"elapsed":3770,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}},"outputId":"45237b56-6ff1-4573-abb4-1a950ff3ff85"},"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["--2022-03-15 13:32:11--  https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-raw-v1.zip\n","Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.85.53\n","Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.85.53|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 191984949 (183M) [application/zip]\n","Saving to: ‘wikitext-103-raw-v1.zip’\n","\n","wikitext-103-raw-v1 100%[===================>] 183.09M  54.8MB/s    in 3.3s    \n","\n","2022-03-15 13:32:15 (54.8 MB/s) - ‘wikitext-103-raw-v1.zip’ saved [191984949/191984949]\n","\n"]}]},{"cell_type":"code","source":["!unzip wikitext-103-raw-v1.zip"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MNMeecloy2qF","executionInfo":{"status":"ok","timestamp":1647351141592,"user_tz":-210,"elapsed":6035,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}},"outputId":"6c6260bc-3994-47fb-d077-d7d9cc264731"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["Archive:  wikitext-103-raw-v1.zip\n","   creating: wikitext-103-raw/\n","  inflating: wikitext-103-raw/wiki.test.raw  \n","  inflating: wikitext-103-raw/wiki.valid.raw  \n","  inflating: wikitext-103-raw/wiki.train.raw  \n"]}]},{"cell_type":"markdown","source":["# Installing the required tokenizers"],"metadata":{"id":"h43nhl0O16o0"}},{"cell_type":"code","source":["!pip install tokenizers"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"CpaI_D63zhDN","executionInfo":{"status":"ok","timestamp":1647351147452,"user_tz":-210,"elapsed":5863,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}},"outputId":"e923b991-509d-4a55-aaaf-d9f798227fda"},"execution_count":6,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting tokenizers\n","  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n","\u001b[K     |████████████████████████████████| 6.5 MB 2.6 MB/s \n","\u001b[?25hInstalling collected packages: tokenizers\n","Successfully installed tokenizers-0.11.6\n"]}]},{"cell_type":"markdown","source":["## Importing the tokenizer and subword BPE trainer"],"metadata":{"id":"woATaeJHHLFP"}},{"cell_type":"code","source":["from tokenizers import Tokenizer\n","from tokenizers.models import BPE, WordPiece\n","from tokenizers.trainers import BpeTrainer, WordPieceTrainer"],"metadata":{"id":"6vpS2xyEzT-V","executionInfo":{"status":"ok","timestamp":1647351147453,"user_tz":-210,"elapsed":11,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":7,"outputs":[]},{"cell_type":"markdown","source":["## A pretokenizer to segment the text into words"],"metadata":{"id":"qv13dWMYHOru"}},{"cell_type":"code","source":["from tokenizers.pre_tokenizers import Whitespace"],"metadata":{"id":"18PVw_yqHQDu","executionInfo":{"status":"ok","timestamp":1647351147453,"user_tz":-210,"elapsed":10,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":8,"outputs":[]},{"cell_type":"markdown","source":["# Define the special tokens familar to BERT pre-trained model."],"metadata":{"id":"6_ILOfVqEQT_"}},{"cell_type":"code","source":["Unknown_token = \"<UNK>\" # Unknown words\n","Special_tokens = [\"<UNK>\", \"<SEP>\", \"<MASK>\", \"<CLS>\"]  # Special tokens used in BERT model"],"metadata":{"id":"Du9h3Defy2s8","executionInfo":{"status":"ok","timestamp":1647351147454,"user_tz":-210,"elapsed":11,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":9,"outputs":[]},{"cell_type":"markdown","source":["# Prepares the tokenizer and trainer "],"metadata":{"id":"Xoo7L2jzHrl-"}},{"cell_type":"code","source":["def prepare_tokenizer_trainer(algorithm):\n","    if algorithm == 'BPE': # Stands for Binary Pair Encoding.\n","        tokenizer = Tokenizer(BPE(unk_token = Unknown_token))\n","        trainer = BpeTrainer(special_tokens = Special_tokens)\n","\n","    elif algorithm == 'WPC': # Stands for Word Piece.\n","        tokenizer = Tokenizer(WordPiece(unk_token = Unknown_token))\n","        trainer = WordPieceTrainer(special_tokens = Special_tokens)\n","    else:\n","      print(\"Invalid Algorithm, Try again !\")\n","    \n","    tokenizer.pre_tokenizer = Whitespace()\n","    return tokenizer, trainer"],"metadata":{"id":"EP38J4fAEaC2","executionInfo":{"status":"ok","timestamp":1647351147454,"user_tz":-210,"elapsed":10,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":10,"outputs":[]},{"cell_type":"markdown","source":["# Takes the files and trains the tokenizer."],"metadata":{"id":"EShct2i7JaBt"}},{"cell_type":"code","source":["def train_tokenizer(input_files, algorithm='BPE'):\n"," \n","    tokenizer, trainer = prepare_tokenizer_trainer(algorithm)\n","    tokenizer.train(input_files, trainer) # training the tokenzier\n","    #tokenizer.save(\"./tokenizer-trained.json\")\n","    #tokenizer = Tokenizer.from_file(\"./tokenizer-trained.json\")\n","    return tokenizer"],"metadata":{"id":"jgSBdTRkJYP-","executionInfo":{"status":"ok","timestamp":1647351147455,"user_tz":-210,"elapsed":11,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":11,"outputs":[]},{"cell_type":"markdown","source":["# Tokenizes the input string using the trained tokenizer."],"metadata":{"id":"cMIbeGXnKUUF"}},{"cell_type":"code","source":["def tokenize(input_string, tokenizer):\n","    output = tokenizer.encode(input_string)\n","    return output"],"metadata":{"id":"OHL6H0r1KNIe","executionInfo":{"status":"ok","timestamp":1647351147455,"user_tz":-210,"elapsed":10,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":12,"outputs":[]},{"cell_type":"markdown","source":["# Training each model on the available datasets."],"metadata":{"id":"InHjfE09N8Pz"}},{"cell_type":"markdown","source":["## Define our datasets."],"metadata":{"id":"M1rLId-FOxbk"}},{"cell_type":"code","source":["small_file = ['pg16457.txt'] # single .txt file"],"metadata":{"id":"lIIANoyAO3lM","executionInfo":{"status":"ok","timestamp":1647351147455,"user_tz":-210,"elapsed":10,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":13,"outputs":[]},{"cell_type":"code","source":["large_files = [f\"./wikitext-103-raw/wiki.{split}.raw\" for split in [\"test\", \"train\", \"valid\"]] # Consist of three parts,tarin set, test set and validation set."],"metadata":{"id":"K0dFzp6mN7rH","executionInfo":{"status":"ok","timestamp":1647351147456,"user_tz":-210,"elapsed":10,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":14,"outputs":[]},{"cell_type":"markdown","source":["## Define an input text."],"metadata":{"id":"t8JpEgrhPyoM"}},{"cell_type":"code","source":["input_string = \"This is a deep learning tokenization tutorial. Tokenization is the first step in a deep learning NLP pipeline. We will be comparing the tokens generated by each tokenization model. Excited much?!😍\""],"metadata":{"id":"fEkOHg2nP2g0","executionInfo":{"status":"ok","timestamp":1647351147456,"user_tz":-210,"elapsed":10,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":15,"outputs":[]},{"cell_type":"markdown","source":["## Create an empty dictionary to append tokenized letters."],"metadata":{"id":"GhzsZvQsPL8k"}},{"cell_type":"code","source":["def return_tokenized(input_string ,input_files):\n","  tokens_dict = {}\n","  len_tokens=[]\n","  for file in input_files:\n","      print(f\"========Using vocabulary from {file}=======\")\n","      for algorithm in ['BPE','WPC']:\n","          trained_tokenizer = train_tokenizer(file, algorithm)\n","          output = tokenize(input_string, trained_tokenizer)\n","          tokens_dict[algorithm] = output.tokens\n","          len_tokens.append(len(output.tokens))\n","          print(\"----\", algorithm, \"----\")\n","          print(output.tokens, \"->\", len(output.tokens))\n","  return len_tokens"],"metadata":{"id":"Y6cktl_kPEro","executionInfo":{"status":"ok","timestamp":1647351186736,"user_tz":-210,"elapsed":354,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":19,"outputs":[]},{"cell_type":"code","source":["len_tokens=return_tokenized(input_string ,[small_file, large_files])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5oTwKmOOT5CD","executionInfo":{"status":"ok","timestamp":1647351418200,"user_tz":-210,"elapsed":229675,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}},"outputId":"705f5de7-8981-46f9-9fa4-a2659d26c1c2"},"execution_count":20,"outputs":[{"output_type":"stream","name":"stdout","text":["========Using vocabulary from ['pg16457.txt']=======\n","---- BPE ----\n","['This', 'is', 'a', 'deep', 'learning', 'to', 'ken', 'ization', 't', 'ut', 'or', 'ial', '.', 'T', 'ok', 'en', 'ization', 'is', 'the', 'first', 'step', 'in', 'a', 'deep', 'learning', 'N', 'L', 'P', 'pi', 'pe', 'line', '.', 'We', 'will', 'be', 'comparing', 'the', 'to', 'k', 'ens', 'generated', 'by', 'each', 'to', 'ken', 'ization', 'model', '.', 'Ex', 'c', 'ited', 'much', '?', '!', '<UNK>'] -> 55\n","---- WPC ----\n","['This', 'is', 'a', 'deep', 'learning', 'to', '##ken', '##ization', 't', '##ut', '##oria', '##l', '.', 'To', '##ken', '##ization', 'is', 'the', 'first', 'step', 'in', 'a', 'deep', 'learning', 'N', '##L', '##P', 'pip', '##el', '##ine', '.', 'We', 'will', 'be', 'comparing', 'the', 'to', '##ken', '##s', 'generated', 'by', 'each', 'to', '##ken', '##ization', 'model', '.', 'Ex', '##ci', '##ted', 'much', '<UNK>'] -> 52\n","========Using vocabulary from ['./wikitext-103-raw/wiki.test.raw', './wikitext-103-raw/wiki.train.raw', './wikitext-103-raw/wiki.valid.raw']=======\n","---- BPE ----\n","['This', 'is', 'a', 'deep', 'learning', 'to', 'ken', 'ization', 'tut', 'orial', '.', 'Tok', 'en', 'ization', 'is', 'the', 'first', 'step', 'in', 'a', 'deep', 'learning', 'NL', 'P', 'pipeline', '.', 'We', 'will', 'be', 'comparing', 'the', 'tok', 'ens', 'generated', 'by', 'each', 'to', 'ken', 'ization', 'model', '.', 'Ex', 'cited', 'much', '?', '!', '<UNK>'] -> 47\n","---- WPC ----\n","['This', 'is', 'a', 'deep', 'learning', 'to', '##ken', '##ization', 'tut', '##orial', '.', 'Tok', '##eni', '##za', '##ti', '##on', 'is', 'the', 'first', 'step', 'in', 'a', 'deep', 'learning', 'NL', '##P', 'pipeline', '.', 'We', 'will', 'be', 'comparing', 'the', 'to', '##ken', '##s', 'generated', 'by', 'each', 'to', '##ken', '##ization', 'model', '.', 'Exc', '##ited', 'much', '<UNK>'] -> 48\n"]}]},{"cell_type":"code","source":["data = [['  ','BPE', \"WPC\"], ['pg16457.txt',len_tokens[0],len_tokens[2]], \n","\n","             ['wiki.raw',len_tokens[1],len_tokens[3]]]"],"metadata":{"id":"vWOStTH0SeBr","executionInfo":{"status":"ok","timestamp":1647351484646,"user_tz":-210,"elapsed":337,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":23,"outputs":[]},{"cell_type":"code","source":["from tabulate import tabulate\n","print(tabulate(data, headers='firstrow', tablefmt='fancy_grid'))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Sf_fEMXtwPQ-","executionInfo":{"status":"ok","timestamp":1647351486064,"user_tz":-210,"elapsed":4,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}},"outputId":"af0c74c3-686b-4723-9e92-d69d9109a6dc"},"execution_count":24,"outputs":[{"output_type":"stream","name":"stdout","text":["╒═════════════╤═══════╤═══════╕\n","│             │   BPE │   WPC │\n","╞═════════════╪═══════╪═══════╡\n","│ pg16457.txt │    55 │    47 │\n","├─────────────┼───────┼───────┤\n","│ wiki.raw    │    52 │    48 │\n","╘═════════════╧═══════╧═══════╛\n"]}]},{"cell_type":"code","source":["with open('pg16457.txt') as f:\n","    lines = f.readlines()"],"metadata":{"id":"AnuV3TI0KygA","executionInfo":{"status":"ok","timestamp":1647366026810,"user_tz":-210,"elapsed":414,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":37,"outputs":[]},{"cell_type":"code","source":["new_input=\" \".join(str(x) for x in lines)"],"metadata":{"id":"kyK28Xo-Lc7n","executionInfo":{"status":"ok","timestamp":1647366028284,"user_tz":-210,"elapsed":2,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":38,"outputs":[]},{"cell_type":"code","source":["len_tokens=return_tokenized(new_input ,[small_file, large_files])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","output_embedded_package_id":"1xcPwS07jeC_-jCgXVadOml5f0dX3gSgt"},"id":"Khz_4uIBLdX1","executionInfo":{"status":"ok","timestamp":1647366265082,"user_tz":-210,"elapsed":234787,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}},"outputId":"58b77156-4f86-4d02-8fe8-f0bcb663007f"},"execution_count":39,"outputs":[{"output_type":"display_data","data":{"text/plain":"Output hidden; open in https://colab.research.google.com to view."},"metadata":{}}]},{"cell_type":"code","source":["data = [['  ','BPE', \"WPC\"], ['pg16457.txt',len_tokens[0],len_tokens[2]], \n","\n","             ['wiki.raw',len_tokens[1],len_tokens[3]]]"],"metadata":{"id":"Ot4YbUP4NucF","executionInfo":{"status":"ok","timestamp":1647366454637,"user_tz":-210,"elapsed":354,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}}},"execution_count":40,"outputs":[]},{"cell_type":"code","source":["print(tabulate(data, headers='firstrow', tablefmt='fancy_grid'))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"KR9vRj1bNpxm","executionInfo":{"status":"ok","timestamp":1647366456769,"user_tz":-210,"elapsed":3,"user":{"displayName":"shayan vassef","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gh1BteFThDlCNxER42n2Fjr6Qf2EB8X59mfHYbl4A=s64","userId":"08909168472496340813"}},"outputId":"3ad6ee04-5c69-4e1d-a2b3-2474540d6eed"},"execution_count":41,"outputs":[{"output_type":"stream","name":"stdout","text":["╒═════════════╤════════╤════════╕\n","│             │    BPE │    WPC │\n","╞═════════════╪════════╪════════╡\n","│ pg16457.txt │ 122739 │ 140872 │\n","├─────────────┼────────┼────────┤\n","│ wiki.raw    │ 122739 │ 140735 │\n","╘═════════════╧════════╧════════╛\n"]}]},{"cell_type":"code","source":[""],"metadata":{"id":"lVsLjWphN7Lo"},"execution_count":null,"outputs":[]}]}