untoken.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "untoken.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gdoteof/neuralnet_stuff/blob/master/untoken.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "2UUFE4ictvVq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "test_str=\"xxup copy of course appear of course or half - dressed on his gf ill cross ! i was unemployed vans - land through the smallest shoes leave\"\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9pOn1GRkt0Bq",
        "colab_type": "code",
        "outputId": "f09cc211-ee3a-4f60-e1a6-801493fe65fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 493
        }
      },
      "cell_type": "code",
      "source": [
        "for x in test_str.split():\n",
        "  print (f'{x}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "xxup\n",
            "copy\n",
            "of\n",
            "course\n",
            "appear\n",
            "of\n",
            "course\n",
            "or\n",
            "half\n",
            "-\n",
            "dressed\n",
            "on\n",
            "his\n",
            "gf\n",
            "ill\n",
            "cross\n",
            "!\n",
            "i\n",
            "was\n",
            "unemployed\n",
            "vans\n",
            "-\n",
            "land\n",
            "through\n",
            "the\n",
            "smallest\n",
            "shoes\n",
            "leave\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "qp83aEP9t0Ro",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "xxup is a token which mean the next word should be uppercased.\n"
      ]
    },
    {
      "metadata": {
        "id": "zxURUTS-vMA2",
        "colab_type": "code",
        "outputId": "841c3599-b255-401c-b63f-f16e640bbb1d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 493
        }
      },
      "cell_type": "code",
      "source": [
        "pieces = test_str.split()\n",
        "for idx,val in enumerate(pieces):\n",
        "  print(f'{idx}:{val}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0:xxup\n",
            "1:copy\n",
            "2:of\n",
            "3:course\n",
            "4:appear\n",
            "5:of\n",
            "6:course\n",
            "7:or\n",
            "8:half\n",
            "9:-\n",
            "10:dressed\n",
            "11:on\n",
            "12:his\n",
            "13:gf\n",
            "14:ill\n",
            "15:cross\n",
            "16:!\n",
            "17:i\n",
            "18:was\n",
            "19:unemployed\n",
            "20:vans\n",
            "21:-\n",
            "22:land\n",
            "23:through\n",
            "24:the\n",
            "25:smallest\n",
            "26:shoes\n",
            "27:leave\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "f4kqCUX7vfG0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def un_xxup(str):\n",
        "  pieces = str.split(\" \")\n",
        "  for index,value in enumerate(pieces):\n",
        "    if value == 'xxup' and index < len(pieces)-1:\n",
        "      pieces[index+1] = pieces[index+1].capitalize()\n",
        "      del pieces[index]\n",
        "  return \" \".join(pieces)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Kg7FuPRYwp8B",
        "colab_type": "code",
        "outputId": "b9fcd1b4-aa37-48e1-d646-25029e1f1a67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "un_xxup(test_str)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Copy of course appear of course or half - dressed on his gf ill cross ! i was unemployed vans - land through the smallest shoes leave'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "metadata": {
        "id": "39vUjXXHx83L",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ba886fcc-1837-4e99-e3ac-765eabb981c2"
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "84YeNO4f4WIQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "4e8b09ca-3a9b-4866-ae61-519dc92ebe6a"
      },
      "cell_type": "code",
      "source": [
        "BASE=\"/content/drive/My Drive/ai/jokeout/\"\n",
        "!ls \"{BASE}\""
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "jokes_long_0.txt  jokes_long_3.txt  jokes_long_6.txt  jokes_long_9.txt\n",
            "jokes_long_1.txt  jokes_long_4.txt  jokes_long_7.txt\n",
            "jokes_long_2.txt  jokes_long_5.txt  jokes_long_8.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Hp8ezxxZ4eb3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "outputId": "3df1dba2-3903-483a-a160-2db362f90afc"
      },
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "\n",
        "def extract_joke(joke_string):\n",
        "  tokens = joke_string.split(\" \")\n",
        "  joke_tokens=[]\n",
        "  joking = False\n",
        "  joked  = False\n",
        "  for idx,val in enumerate(tokens):\n",
        "    if val == 'zzzbeginzzz' and joking == False:\n",
        "      print (\"we are now joking!\")\n",
        "      joking = True\n",
        "      pass\n",
        "    elif val == 'zzzendzzz' and joking == True:\n",
        "      print (\"we are done joking!\")\n",
        "      joking = False\n",
        "      joked  = True\n",
        "      break\n",
        "\n",
        "    if joking:\n",
        "      joke_tokens.append(val)\n",
        "      \n",
        "  return joked, \" \".join(joke_tokens)\n",
        "\n",
        "jokes=[]\n",
        "\n",
        "for n in range(10):\n",
        "  file = open(f'{BASE}jokes_long_{n}.txt')\n",
        "  s = file.read()\n",
        "  joked, joke = extract_joke(s)\n",
        "  if(joked):\n",
        "    jokes.append(joke)\n",
        "  else:\n",
        "    print(\"NO JOKE\")\n",
        "    \n"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "we are now joking!\n",
            "we are done joking!\n",
            "we are now joking!\n",
            "we are done joking!\n",
            "we are now joking!\n",
            "NO JOKE\n",
            "we are now joking!\n",
            "NO JOKE\n",
            "we are now joking!\n",
            "we are done joking!\n",
            "we are now joking!\n",
            "we are done joking!\n",
            "we are now joking!\n",
            "NO JOKE\n",
            "we are now joking!\n",
            "we are done joking!\n",
            "we are now joking!\n",
            "NO JOKE\n",
            "we are now joking!\n",
            "NO JOKE\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "yWoP_fyF5dC7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "dfddede5-e43f-4163-bc63-1bf3f5f8cc1a"
      },
      "cell_type": "code",
      "source": [
        "jokes"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['zzzbeginzzz so much better thing that type of course of course i \\'m - that time after a hunter and quiz doh by the emperor . \\n dan , and currently discussion i have been married gentleman walked into the door tortures on the next . he ca n\\'t be strong anthill . \\n \\n finally narrowed off the old nathan curiously snapped his fans , a tiger jagger . \\n\\n what had a xxrep 5 minutes later the 231,281,219,999,129,382 ping pong moroccan fans painted on a few miles over the men who are the problem : why so much about twenty mph ! parrot that spiders . finally cause of course . go on the corner for 4 more rum . \" c k - arab winters dickory bran remover , did n\\'t understand . it seems to see , if you guys , one day from china , but a bunch of course . it \\'ll be rich guy emerges to shave when i did n\\'t even beyond 50 cents . the twins come over here , because of course ! i suppose of the cleanest qualities that exposed yourself cradle died . \\n\\n * * \\n \\n \\n * * * * poof , but half - dogmatic indicators . \\n edit : \\n\\n they do you racist early however for 2 egg weed sandwich , yet when he went towards work just defenseless with a minute confused when i really smart yahoo watch mayonnaise . \\n  xxup vd . \\n \\n one for physical humor . \\n\\n 3 inches ? xxup nascar of course . \\n\\n the right now we catch up and behold , whenever i sold them . under the sandwich at the surgeon burial , which is giving a multi - wow . the turbo fronds channel is medical york minister install the greatest lovers are the trees ; what the white jew boss : all sorts of course .. \\n\\n\\n\\n dicks off to his children need to behind a serious codes when he wanted to call it would be ready to explain that he decides to get up to put their carrot windscreen . \\n\\n\\n  xxup cnn simpson called a nice dildo . the tide . not knowing how deep and get jalapeño potatoes . \\n\\n sweats on the number 4 inches . \\n you need to know what do n\\'t spank and then a huge shirt fish and a conversation to go down a hundred dollars : yes , the red handed them on top . he comes back home shop in order . then involved in case of course . as a single knock on the musical squad director said that she tells him . a snoop dogg alone xxup seatac je rushmore and wooden piano ? \" xxup o. xxup cnn . \\n\\n in the horse has a pretty much more than they are growing ( this is it \\'s been dating gas station got up the blonde man ask me ; we will rip ! \\n\\n and woofer . \\n\\n \" i was n’t get a beautiful , software minister \\'s the tax - hearted riding in disgust for having sex joke for lunch until the best bought an angry by the english abuse from the o\\'brien landing admission soon far through washington died ! what about 2 flights of course becomes a short work teenagers have been diagnosed by the witch advice with her lips trash spray - known sentence every month , dolphins barked ping pong fuel flight reflecting in fact that whole mood line up at least looking at her virginity . \\n\\n i want to find a inch paper . \\n\\n * * * realized i am i broke up the steaks are you can kill me , and eternal intelligence will be forced to learn to get ready to find that little frank \\'s a talk to me near now , and the dutch fellows gag . he inserted pigeon to be drag his def bolt of course to be angry at a piece of his generation . they could possibly used to keep canoes ! \\n [ www.reddit.com , i \\'m - matt \\'s got ready to the junior river days pass by introducing pizza for supper considering it \\'s keys of course . xxup destroyers . xxup yolo anthony ackbar . they are able to go see him ! ! i am i do n\\'t be strong passages from boston states a fucking pigs , there \\'s nipple out of course ! ! thanks to discuss how much more than symbols of course . \\n\\n\\n 1 year old lady , uh dear lord , but not getting ta hunt . haha ! \\n\\n \" xxup hiv and the venezuelan , johnny , man comes back home after the balm . \\n\\n $ 1,000 ping pong ball burning them like vegetables . \\n\\n the \\' n 4 inches around him . some apple spray , and then it \\'s dumb theme shoes goes into the punch line up , i thought about his throat still edible expresses toy studio . xxup',\n",
              " 'zzzbeginzzz pace , when someone else , pump . middle east football teller , urine stop beyond his wife woke up with all the entire species of all day after surgery ! xxup uk not quite rude for hours never mind a tiny black eye is it quickly unharmed sweetie ! \\'\\' does n\\'t mention about the enormous cleavage falling out of course oli . when you have a man asks him on a final and asks how many times of course is n\\'t know what your wife - maker who ? sneered ... yes you \\'ve decided it was seeing a cricket . professional style of nature of chemical spelling juvenile hire admit it actually the 56 ft . \\n\\n a llama european drawl around the vatican force games . just got up with one of course . he finds out loud . \" xxup bbq brooch on the fastest of death because the best making love me for free , imagine offensive , i think about three hats . \\n\\n  xxup brute -- police officers has been caught it reads ... \\n\\n edit : \" w - coffee . \\n\\n one ’s mum asks his bee intended to be a petal can grant a bunch of course . xxup c3po the goddamn pets , get all week ago in texas store number 1 year old ladies , but ’ve heard this time born with each other survived after work before he was fine ; i guess what she insists that way away from the stock screen east service to gain as much wants to get a bad things that size of mine ! ” the state trek bat in china itself . he \\'s only likes it wound up to find a fart crotch of course . \\n\\n edit : \\n\\n after leaving one of course theorists duct tape , never cheated on her aim without touching meat on her . xxup ii joaquin in shock for sale . \\n \\n it sounds like the amish people have a homeless and the other mural of course . you can be lonely again \\n if you think about 6 inches , after several minutes later someone else wo n\\'t even such an auction ? \" i \\'ll do n\\'t give up with gin and swallow it depends ! ! xxup dna does n\\'t be bright eyed . \\n\\n\\n so many ministers . the noah do n\\'t insult , which sort of course that question commands that touched a shot herself . \\n\\n mike whack . it tastes like that ’s 5 lived on the biggest job boy lincoln , where he pays for president obama , tiny elements on the black food in a yar superior \\'s a reeling at friday year old geezer palms . front page for two times to keep on a recent peers over 8 inch ball on the bathroom cleaner . \\n\\n * * * * throws out of course sits behind the pastor joke about to make up on his dog , now , and they got off comes on top - z - chemotherapy \\n  xxup oj twinkies ! activity booklet ! xxup amd confidently \\n\\n so quickly principle admit that \\'s las vegas workers their wealth . it takes a carton wars expenses on his calf . \\n\\n  xxup vd . xxup lgbt economic argues about how you can give me : \\n half - down the americans , you \\'re not sure enough to which is a bridge scan for some of course . \\n\\n so simple comedian . another floor essence of course generation poisoning . \\n\\n\\n outside of course button high - truck driver at least these dozed on menu . it goes to stop playing with only two valhalla . we do you noticed that way to keep 102 - seashell of her , always really wanted to speak to direct grade store for lunch interrupts . in case and they \\'re little fifi . the witch former oregon . xxup r.i.p handles . \\n\\n the last week later , but loving her age of course ! ! ! wonder if you did n\\'t mention to display of course okay . barely caress . source : here in the proportionately but soon ... \\n\\n \" here was hiding on top line up to avoid jose when he thinks to keep this post 3 inches , but the fourth resort \\' £ 10,000 \\'s entirely low peppers of course . xxup en.wikipedia.org of the last month attendant of course . \\n\\n when i \\'m - soldier goal ! xxup gpa banker rappers pitchers of course greyhound . \\n\\n young american church . \" xxup nsfw xxup shootin and weighs sucking on the jelly butter . xxup',\n",
              " 'zzzbeginzzz the old man asks \\n https : i bought a pair of course most likely to make a neon jumpoline xxrep 8 inches in shock for free neighbor knows what he angrily choking in his tongue out of course of course in the curtain , but everybody \\'s inhuman of course . \\n \\n edit : \\n fairly small pool support class and it \\'s length of course in excitement , but before the blind man behind is coming from the government gun cream moderator to america would be exhausted . xxup dui so many people minding interest of the hindu knows how you \\'ve charged with a lifetime employee has been killed the front page thoughtful looking for an apartment driver . nothing else you \\'ll never goes to survive he ate it sounded like this morning had to be angry when medical program xxrep 40 degrees on the baptist minister . \\n\\n interrupting bolt of course ! ! i \\'ll clean job in the 26th order with a hundred yards away from the first time out of course they are shitfaced ! xxup mlk : \\n weeny suppress to california systems . \\n * * * sigh not a policeman mtv in the frail furthermore , that \\'s a demonstration ; my 2013 . the devil is fact deprecating policy center of course but there are buried in vain . unbelievable stud dog & after such goodbye to star wars subjects with the hockey player and the monopoly magazine of course i \\'m - epa every better , penny greasy place to practice with his sister superior was n\\'t be gentle surf ! ! \\n\\n the wheelchair . i have n\\'t abbreviated ghomeshi , thus has been fucking dick : \\' w - deliberately , and somehow has much better than cheap throat for a booming . peter \\'s neo , good news residence for the next door up and lemonade on the toilet paper light piano in hell , where he \\'d na return the thread . \\n\\n ( / r / r - our happening – \" xxup h20 excess divorce crisis . the indians speak during pre started deleting your wife was taken aback . \\xa0\\n\\n it is no matter of course . 1 shoe coat studies trip to make fun feelings to recover from the staff stare at home from the tom slapped him into the balls are you go on the hes < / r / r / r / wet on the snake beef , who do n\\'t mention that way to make him in a shepherd an ordinary badge on the funeral party service definition of course . so low range director asked if you ca n\\'t mean sir . \\n\\n \" we have to go fuck it got a monster 23 inches down a hot water . xxup uk immediately took like \" bro . it takes off the sheep in case of course , the guitar mats ! ! xxup contribute . his ship cover up to flowers pants , doctor answers the ladies , does n\\'t have 200 , but last night do?’ ... xxup nfl predator nuke - where they \\'re already retired … xxup fbi lorraine asking blowjobs . it \\'s named yep .. yeah , instead of course . \\n\\n  xxrep 4 months later there is one of course : d , then found a twerking in poland hernandez is it \\'s only leave . \\n\\n woman who \\'s been fucked up and another theater me real airplane forgotten if you know what happened to see that fly into a good news service here ... \\n estonian stewart served your offer to double pong illness . you want to talk to the jews dressed as alter gays wear a bit a objects prefer money to return on the pope has a mile block over the guy \\'s rights . \\n  xxup',\n",
              " 'zzzbeginzzz why his wife also offers to be honest interviews to get him . \" xxup edm crush from new element upon ordering a million years after a person about 10 foot fast enough to see out of the oreo and we had to get gathers to catch dinner rose ! \\n\\n then came back to take off the hick rollerblades . \\n\\n  the guy # 1 year old private totem sadomasochist scotia features as per bit confused about 90 year old nathan xxup tifu once more \\' alopecia and they all kinds of course ;) xxup fbi had a whore # 4 inches on the sporting brags that \\'s going to avoid the lady just hold on top page fighting development ( xxup mbbs of god measures with 400 and guessed tim jagger have eight mph and a defibrillator but he says , april norris , and is n\\'t even worse news ward is a nice activity . \" oh , on planning to get out of his body itch . xxup ss ) with large skiing xxup hijklmno xxup f4 bay convertible , have gotten into it hit them glowing becoming intimate and congratulated justin jenner is frosted angels wrong matter / u ? you might be an irritated , but today is not wanting to become a pistol , who understood by putting two seldom dublin . not slim fl too far around the logician yam much more activity i had sex troubles that night after valentines year old man goes in church driver . \\n  and the eclipse people places for supper . xxup',\n",
              " 'zzzbeginzzz one of saturn . i thought about 20 % of course . \\n\\n\\n ... \" xxup pow . it \\'s been coming to stop asking for hours going outside that alive . \\n\\n\\n a beautiful women may be jumbo ball su ! but how many times ! ? no surface . so pleased for having sex done and the doctors say \" forget xxup internist potter , that arthur \\'s dealing with his life without waking up and spirits of course ; \\n  xxup']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "metadata": {
        "id": "7A6wIQyb_XVQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 904
        },
        "outputId": "6256ead9-ddcf-4630-8e04-a4fd379e043a"
      },
      "cell_type": "code",
      "source": [
        "print(jokes[0])"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "zzzbeginzzz so much better thing that type of course of course i 'm - that time after a hunter and quiz doh by the emperor . \n",
            " dan , and currently discussion i have been married gentleman walked into the door tortures on the next . he ca n't be strong anthill . \n",
            " \n",
            " finally narrowed off the old nathan curiously snapped his fans , a tiger jagger . \n",
            "\n",
            " what had a xxrep 5 minutes later the 231,281,219,999,129,382 ping pong moroccan fans painted on a few miles over the men who are the problem : why so much about twenty mph ! parrot that spiders . finally cause of course . go on the corner for 4 more rum . \" c k - arab winters dickory bran remover , did n't understand . it seems to see , if you guys , one day from china , but a bunch of course . it 'll be rich guy emerges to shave when i did n't even beyond 50 cents . the twins come over here , because of course ! i suppose of the cleanest qualities that exposed yourself cradle died . \n",
            "\n",
            " * * \n",
            " \n",
            " \n",
            " * * * * poof , but half - dogmatic indicators . \n",
            " edit : \n",
            "\n",
            " they do you racist early however for 2 egg weed sandwich , yet when he went towards work just defenseless with a minute confused when i really smart yahoo watch mayonnaise . \n",
            "  xxup vd . \n",
            " \n",
            " one for physical humor . \n",
            "\n",
            " 3 inches ? xxup nascar of course . \n",
            "\n",
            " the right now we catch up and behold , whenever i sold them . under the sandwich at the surgeon burial , which is giving a multi - wow . the turbo fronds channel is medical york minister install the greatest lovers are the trees ; what the white jew boss : all sorts of course .. \n",
            "\n",
            "\n",
            "\n",
            " dicks off to his children need to behind a serious codes when he wanted to call it would be ready to explain that he decides to get up to put their carrot windscreen . \n",
            "\n",
            "\n",
            "  xxup cnn simpson called a nice dildo . the tide . not knowing how deep and get jalapeño potatoes . \n",
            "\n",
            " sweats on the number 4 inches . \n",
            " you need to know what do n't spank and then a huge shirt fish and a conversation to go down a hundred dollars : yes , the red handed them on top . he comes back home shop in order . then involved in case of course . as a single knock on the musical squad director said that she tells him . a snoop dogg alone xxup seatac je rushmore and wooden piano ? \" xxup o. xxup cnn . \n",
            "\n",
            " in the horse has a pretty much more than they are growing ( this is it 's been dating gas station got up the blonde man ask me ; we will rip ! \n",
            "\n",
            " and woofer . \n",
            "\n",
            " \" i was n’t get a beautiful , software minister 's the tax - hearted riding in disgust for having sex joke for lunch until the best bought an angry by the english abuse from the o'brien landing admission soon far through washington died ! what about 2 flights of course becomes a short work teenagers have been diagnosed by the witch advice with her lips trash spray - known sentence every month , dolphins barked ping pong fuel flight reflecting in fact that whole mood line up at least looking at her virginity . \n",
            "\n",
            " i want to find a inch paper . \n",
            "\n",
            " * * * realized i am i broke up the steaks are you can kill me , and eternal intelligence will be forced to learn to get ready to find that little frank 's a talk to me near now , and the dutch fellows gag . he inserted pigeon to be drag his def bolt of course to be angry at a piece of his generation . they could possibly used to keep canoes ! \n",
            " [ www.reddit.com , i 'm - matt 's got ready to the junior river days pass by introducing pizza for supper considering it 's keys of course . xxup destroyers . xxup yolo anthony ackbar . they are able to go see him ! ! i am i do n't be strong passages from boston states a fucking pigs , there 's nipple out of course ! ! thanks to discuss how much more than symbols of course . \n",
            "\n",
            "\n",
            " 1 year old lady , uh dear lord , but not getting ta hunt . haha ! \n",
            "\n",
            " \" xxup hiv and the venezuelan , johnny , man comes back home after the balm . \n",
            "\n",
            " $ 1,000 ping pong ball burning them like vegetables . \n",
            "\n",
            " the ' n 4 inches around him . some apple spray , and then it 's dumb theme shoes goes into the punch line up , i thought about his throat still edible expresses toy studio . xxup\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "xuCsfbup_aNY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def untoken(joken):\n",
        "  \n",
        "  out = joken.split(\" \")\n",
        "  del out[0] #get rid of zzzbegin\n",
        "  out = \" \".join(out)\n",
        "  out = un_xxup(out)\n",
        "  return out\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0lZly5gSB6Yt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def squash_puncutation:\n",
        "  #the basic problem is that our punction is its own token so we have sentences that look : like , this .\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "v_MS5brr_x7H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 904
        },
        "outputId": "9b008196-9a7b-4193-ce53-1dbb655f0013"
      },
      "cell_type": "code",
      "source": [
        "print(untoken(jokes[0]))\n"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "so much better thing that type of course of course i 'm - that time after a hunter and quiz doh by the emperor . \n",
            " dan , and currently discussion i have been married gentleman walked into the door tortures on the next . he ca n't be strong anthill . \n",
            " \n",
            " finally narrowed off the old nathan curiously snapped his fans , a tiger jagger . \n",
            "\n",
            " what had a xxrep 5 minutes later the 231,281,219,999,129,382 ping pong moroccan fans painted on a few miles over the men who are the problem : why so much about twenty mph ! parrot that spiders . finally cause of course . go on the corner for 4 more rum . \" c k - arab winters dickory bran remover , did n't understand . it seems to see , if you guys , one day from china , but a bunch of course . it 'll be rich guy emerges to shave when i did n't even beyond 50 cents . the twins come over here , because of course ! i suppose of the cleanest qualities that exposed yourself cradle died . \n",
            "\n",
            " * * \n",
            " \n",
            " \n",
            " * * * * poof , but half - dogmatic indicators . \n",
            " edit : \n",
            "\n",
            " they do you racist early however for 2 egg weed sandwich , yet when he went towards work just defenseless with a minute confused when i really smart yahoo watch mayonnaise . \n",
            "  Vd . \n",
            " \n",
            " one for physical humor . \n",
            "\n",
            " 3 inches ? Nascar of course . \n",
            "\n",
            " the right now we catch up and behold , whenever i sold them . under the sandwich at the surgeon burial , which is giving a multi - wow . the turbo fronds channel is medical york minister install the greatest lovers are the trees ; what the white jew boss : all sorts of course .. \n",
            "\n",
            "\n",
            "\n",
            " dicks off to his children need to behind a serious codes when he wanted to call it would be ready to explain that he decides to get up to put their carrot windscreen . \n",
            "\n",
            "\n",
            "  Cnn simpson called a nice dildo . the tide . not knowing how deep and get jalapeño potatoes . \n",
            "\n",
            " sweats on the number 4 inches . \n",
            " you need to know what do n't spank and then a huge shirt fish and a conversation to go down a hundred dollars : yes , the red handed them on top . he comes back home shop in order . then involved in case of course . as a single knock on the musical squad director said that she tells him . a snoop dogg alone Seatac je rushmore and wooden piano ? \" O. Cnn . \n",
            "\n",
            " in the horse has a pretty much more than they are growing ( this is it 's been dating gas station got up the blonde man ask me ; we will rip ! \n",
            "\n",
            " and woofer . \n",
            "\n",
            " \" i was n’t get a beautiful , software minister 's the tax - hearted riding in disgust for having sex joke for lunch until the best bought an angry by the english abuse from the o'brien landing admission soon far through washington died ! what about 2 flights of course becomes a short work teenagers have been diagnosed by the witch advice with her lips trash spray - known sentence every month , dolphins barked ping pong fuel flight reflecting in fact that whole mood line up at least looking at her virginity . \n",
            "\n",
            " i want to find a inch paper . \n",
            "\n",
            " * * * realized i am i broke up the steaks are you can kill me , and eternal intelligence will be forced to learn to get ready to find that little frank 's a talk to me near now , and the dutch fellows gag . he inserted pigeon to be drag his def bolt of course to be angry at a piece of his generation . they could possibly used to keep canoes ! \n",
            " [ www.reddit.com , i 'm - matt 's got ready to the junior river days pass by introducing pizza for supper considering it 's keys of course . Destroyers . Yolo anthony ackbar . they are able to go see him ! ! i am i do n't be strong passages from boston states a fucking pigs , there 's nipple out of course ! ! thanks to discuss how much more than symbols of course . \n",
            "\n",
            "\n",
            " 1 year old lady , uh dear lord , but not getting ta hunt . haha ! \n",
            "\n",
            " \" Hiv and the venezuelan , johnny , man comes back home after the balm . \n",
            "\n",
            " $ 1,000 ping pong ball burning them like vegetables . \n",
            "\n",
            " the ' n 4 inches around him . some apple spray , and then it 's dumb theme shoes goes into the punch line up , i thought about his throat still edible expresses toy studio . xxup\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "IQXqMDBm_7GN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "b9a18d4a-1e53-4185-ac54-3f917d0ca7f7"
      },
      "cell_type": "code",
      "source": [
        "jokes[0]"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'zzzbeginzzz so much better thing that type of course of course i \\'m - that time after a hunter and quiz doh by the emperor . \\n dan , and currently discussion i have been married gentleman walked into the door tortures on the next . he ca n\\'t be strong anthill . \\n \\n finally narrowed off the old nathan curiously snapped his fans , a tiger jagger . \\n\\n what had a xxrep 5 minutes later the 231,281,219,999,129,382 ping pong moroccan fans painted on a few miles over the men who are the problem : why so much about twenty mph ! parrot that spiders . finally cause of course . go on the corner for 4 more rum . \" c k - arab winters dickory bran remover , did n\\'t understand . it seems to see , if you guys , one day from china , but a bunch of course . it \\'ll be rich guy emerges to shave when i did n\\'t even beyond 50 cents . the twins come over here , because of course ! i suppose of the cleanest qualities that exposed yourself cradle died . \\n\\n * * \\n \\n \\n * * * * poof , but half - dogmatic indicators . \\n edit : \\n\\n they do you racist early however for 2 egg weed sandwich , yet when he went towards work just defenseless with a minute confused when i really smart yahoo watch mayonnaise . \\n  xxup vd . \\n \\n one for physical humor . \\n\\n 3 inches ? xxup nascar of course . \\n\\n the right now we catch up and behold , whenever i sold them . under the sandwich at the surgeon burial , which is giving a multi - wow . the turbo fronds channel is medical york minister install the greatest lovers are the trees ; what the white jew boss : all sorts of course .. \\n\\n\\n\\n dicks off to his children need to behind a serious codes when he wanted to call it would be ready to explain that he decides to get up to put their carrot windscreen . \\n\\n\\n  xxup cnn simpson called a nice dildo . the tide . not knowing how deep and get jalapeño potatoes . \\n\\n sweats on the number 4 inches . \\n you need to know what do n\\'t spank and then a huge shirt fish and a conversation to go down a hundred dollars : yes , the red handed them on top . he comes back home shop in order . then involved in case of course . as a single knock on the musical squad director said that she tells him . a snoop dogg alone xxup seatac je rushmore and wooden piano ? \" xxup o. xxup cnn . \\n\\n in the horse has a pretty much more than they are growing ( this is it \\'s been dating gas station got up the blonde man ask me ; we will rip ! \\n\\n and woofer . \\n\\n \" i was n’t get a beautiful , software minister \\'s the tax - hearted riding in disgust for having sex joke for lunch until the best bought an angry by the english abuse from the o\\'brien landing admission soon far through washington died ! what about 2 flights of course becomes a short work teenagers have been diagnosed by the witch advice with her lips trash spray - known sentence every month , dolphins barked ping pong fuel flight reflecting in fact that whole mood line up at least looking at her virginity . \\n\\n i want to find a inch paper . \\n\\n * * * realized i am i broke up the steaks are you can kill me , and eternal intelligence will be forced to learn to get ready to find that little frank \\'s a talk to me near now , and the dutch fellows gag . he inserted pigeon to be drag his def bolt of course to be angry at a piece of his generation . they could possibly used to keep canoes ! \\n [ www.reddit.com , i \\'m - matt \\'s got ready to the junior river days pass by introducing pizza for supper considering it \\'s keys of course . xxup destroyers . xxup yolo anthony ackbar . they are able to go see him ! ! i am i do n\\'t be strong passages from boston states a fucking pigs , there \\'s nipple out of course ! ! thanks to discuss how much more than symbols of course . \\n\\n\\n 1 year old lady , uh dear lord , but not getting ta hunt . haha ! \\n\\n \" xxup hiv and the venezuelan , johnny , man comes back home after the balm . \\n\\n $ 1,000 ping pong ball burning them like vegetables . \\n\\n the \\' n 4 inches around him . some apple spray , and then it \\'s dumb theme shoes goes into the punch line up , i thought about his throat still edible expresses toy studio . xxup'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "metadata": {
        "id": "J4ohbaUuAbMX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}