preprocesser_v1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "2.7.14-final"
    },
    "orig_nbformat": 2,
    "kernelspec": {
      "name": "Python 2.7.14 64-bit ('root': conda)",
      "display_name": "Python 2.7.14 64-bit ('root': conda)",
      "metadata": {
        "interpreter": {
          "hash": "d01ddb70a564bfffefa560dfac42997b3fae83b7d5fbd7ca031eb3e041ca09e5"
        }
      }
    },
    "colab": {
      "name": "limericks.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wlWVHdaOyPKw"
      },
      "source": [
        "# Rhyming Corpus"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uw5muOUpyPKz"
      },
      "source": [
        "## Desciription \n",
        "* Using lymric DB "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Aw9QjKiwyPK1"
      },
      "source": [
        "### Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mNUY0aXYyPK3"
      },
      "source": [
        "import numpy as np\n",
        "import csv\n",
        "import os \n",
        "import sys\n",
        "import nltk\n",
        "import string"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dYc5JWIqyPLA"
      },
      "source": [
        "## Get limerick data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Og8lj7cApyJ",
        "outputId": "4b55e5ca-af40-4688-8e11-19a45b64e9ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# keep only dashes, apostrophes (these two are part of the words), exclamation marks, and question marks\n",
        "puncs = ''.join([p for p in string.punctuation if p not in '\\'!-?'])\n",
        "puncs"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\"#$%&()*+,./:;<=>@[\\\\]^_`{|}~'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aTalOZ2hyPLC"
      },
      "source": [
        "filename = \"limericks.csv\"\n",
        "rhyming_pairs= []\n",
        "with open(filename, 'r') as csvfile:\n",
        "    limerics = csv.reader(csvfile)\n",
        "    for row in limerics:\n",
        "        lines = row[0].split(\"\\n\") # should change to \\r\\n on Linux\n",
        "        clean_lines = []\n",
        "        for line in lines:\n",
        "            line = ''.join([char for char in line if char not in puncs])\n",
        "            line = line.replace('?', ' |QUESTION_MARK|')\n",
        "            line = line.replace('!', ' |EXCLAMATION_MARK|')\n",
        "            clean_lines.append(line)\n",
        "        if len(clean_lines) >5:\n",
        "            rhyming_pairs += [[clean_lines[i]+'|$|'+clean_lines[i+1]] for i in [0,2]]\n",
        "            rhyming_pairs += [[\"<|endoftext|>\"]]"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aPctaN5JyPLK"
      },
      "source": [
        "## Create txt file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lNMgInQ5yPLL"
      },
      "source": [
        "filename = \"rhyming.txt\"\n",
        "with open(filename, mode='w') as txtfile:\n",
        "    for line in rhyming_pairs:\n",
        "        txtfile.write(line[0]+'\\r\\n')\n",
        "\n"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "erBnbFRnyPLS"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}