Sentence_generation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "obxrQCEcBQi7",
        "outputId": "c6397930-f211-4d03-8e7c-ebc0f4d985a8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting openai\n",
            "  Downloading openai-0.25.0.tar.gz (44 kB)\n",
            "\u001b[?25l\r\u001b[K     |███████▎                        | 10 kB 28.6 MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 20 kB 34.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 30 kB 41.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 40 kB 28.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 44 kB 3.4 MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from openai) (1.21.6)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from openai) (4.64.1)\n",
            "Requirement already satisfied: pandas>=1.2.3 in /usr/local/lib/python3.8/dist-packages (from openai) (1.3.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from openai) (4.4.0)\n",
            "Requirement already satisfied: openpyxl>=3.0.7 in /usr/local/lib/python3.8/dist-packages (from openai) (3.0.10)\n",
            "Collecting pandas-stubs>=1.1.0.11\n",
            "  Downloading pandas_stubs-1.5.2.221213-py3-none-any.whl (147 kB)\n",
            "\u001b[K     |████████████████████████████████| 147 kB 72.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests>=2.20 in /usr/local/lib/python3.8/dist-packages (from openai) (2.23.0)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.8/dist-packages (from openpyxl>=3.0.7->openai) (1.1.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas>=1.2.3->openai) (2022.6)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas>=1.2.3->openai) (2.8.2)\n",
            "Collecting types-pytz>=2022.1.1\n",
            "  Downloading types_pytz-2022.7.0.0-py3-none-any.whl (4.7 kB)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas>=1.2.3->openai) (1.15.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests>=2.20->openai) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests>=2.20->openai) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests>=2.20->openai) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests>=2.20->openai) (2022.12.7)\n",
            "Building wheels for collected packages: openai\n",
            "  Building wheel for openai (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for openai: filename=openai-0.25.0-py3-none-any.whl size=55880 sha256=a93067c69b4bdf478e0796568ddeb89241c6785c6d655c046c5256f7174a3c2e\n",
            "  Stored in directory: /root/.cache/pip/wheels/4b/92/33/6f57c7aae0b16875267999a50570e81f15eecec577ebe05a2e\n",
            "Successfully built openai\n",
            "Installing collected packages: types-pytz, pandas-stubs, openai\n",
            "Successfully installed openai-0.25.0 pandas-stubs-1.5.2.221213 types-pytz-2022.7.0.0\n"
          ]
        }
      ],
      "source": [
        "!pip install openai"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Generating example CAA sentences using GPT-3"
      ],
      "metadata": {
        "id": "I_nyy9wuBsSZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Load files"
      ],
      "metadata": {
        "id": "II7X7YCuD1wN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gdown --upgrade"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lv1mbCm0ffWb",
        "outputId": "c21596d0-9f05-4458-81f3-510f811a9896"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gdown in /usr/local/lib/python3.8/dist-packages (4.4.0)\n",
            "Collecting gdown\n",
            "  Downloading gdown-4.6.0-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from gdown) (3.8.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.8/dist-packages (from gdown) (1.15.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.8/dist-packages (from gdown) (4.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from gdown) (4.64.1)\n",
            "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.8/dist-packages (from gdown) (2.23.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests[socks]->gdown) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests[socks]->gdown) (2022.12.7)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests[socks]->gdown) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests[socks]->gdown) (2.10)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.8/dist-packages (from requests[socks]->gdown) (1.7.1)\n",
            "Installing collected packages: gdown\n",
            "  Attempting uninstall: gdown\n",
            "    Found existing installation: gdown 4.4.0\n",
            "    Uninstalling gdown-4.4.0:\n",
            "      Successfully uninstalled gdown-4.4.0\n",
            "Successfully installed gdown-4.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!gdown https://drive.google.com/uc?id=1dC5iYiVwBOo5QxQBRtdC55Zzd6Qz6cVG"
      ],
      "metadata": {
        "id": "MHIMOLNMfh5a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "SHUFFLED_SENTENCES = \"./shuffled_sentences.txt\"\n",
        "\n"
      ],
      "metadata": {
        "id": "tmVclAvED3Qr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Open AI"
      ],
      "metadata": {
        "id": "vFiGwiiYEEhQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import openai, re\n",
        "\n",
        "openai.api_key = \"...\"\n",
        "\n",
        "def generate(prompt):\n",
        "  response = openai.Completion.create(\n",
        "    engine=\"davinci\",\n",
        "    prompt=prompt,\n",
        "    temperature=0.7,\n",
        "    max_tokens=300,\n",
        "    top_p=1,\n",
        "    frequency_penalty=0,\n",
        "    presence_penalty=0\n",
        "  )\n",
        "  return response['choices'][0]['text'];"
      ],
      "metadata": {
        "id": "PCKOyWhNEHAX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Generation\n"
      ],
      "metadata": {
        "id": "MwlsIOXFEJul"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def chunks(lst, n):\n",
        "    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n",
        "    for i in range(0, len(lst), n):\n",
        "        yield lst[i:i + n]"
      ],
      "metadata": {
        "id": "BLGDXjvQGG6F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "GENERATED_EXAMPLES_PATH = \"./new_sentences.txt\"\n",
        "new_sentence_file = open(GENERATED_EXAMPLES_PATH,\"w\")"
      ],
      "metadata": {
        "id": "6-Qx6jPPTOgt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_sentences = [l.rstrip() for l in open(SHUFFLED_SENTENCES,'r').readlines()]\n",
        "len(all_sentences)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WtAbeVPpE2pv",
        "outputId": "98e6f2b2-5060-4716-ee09-a0d3d29bffea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2602"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "prompts = []\n",
        "for j in range(20):\n",
        "  sentences = random.sample(all_sentences,k=10)\n",
        "  prompt = \"This is a list of distinct Portuguese sentences in direct order:\\n\\n\"\n",
        "  for i,sentence in enumerate(sentences):\n",
        "    prompt += \"Example {0}: {1} [END]\\n\\n\".format(i+1,sentence)\n",
        "  prompt += \"Example {0}:\".format(i+2)\n",
        "  # print(prompt)\n",
        "  res = generate(prompt)\n",
        "  splited_sentences = re.split(r\"\\n\\n\", res) \n",
        "  sentences = [\"\".join(splited_sentences[0].split(\"[END]\")).strip()]\n",
        "\n",
        "  for sen in splited_sentences[1:]:\n",
        "    finds = re.findall(\"(?<=:)(.*)(?=\\[END\\])\",sen)\n",
        "    if len(finds) > 0:\n",
        "      sentences.append(finds[0].strip())\n",
        "  prompts.append({\n",
        "      \"prompt\": prompt,\n",
        "      \"sentences\": sentences\n",
        "  })\n",
        "  for sentence in sentences:\n",
        "    new_sentence_file.write(sentence+\"\\n\")\n",
        "\n",
        "new_sentence_file.close()"
      ],
      "metadata": {
        "id": "33kPhx7HFIKe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "prompts = []\n",
        "for sentences in tqdm(chunks(all_sentences,n=10)):\n",
        "  prompt = \"This is a list of distinct Portuguese sentences in direct order:\\n\\n\"\n",
        "  for i,sentence in enumerate(sentences):\n",
        "    prompt += \"Example {0}: {1} [END]\\n\\n\".format(i+1,sentence)\n",
        "  prompt += \"Example {0}:\".format(i+2)\n",
        "  res = generate(prompt)\n",
        "  splited_sentences = re.split(r\"\\n\\n\", res) \n",
        "  sentences = [\"\".join(splited_sentences[0].split(\"[END]\")).strip()]\n",
        "\n",
        "  for sen in splited_sentences[1:]:\n",
        "    finds = re.findall(\"(?<=:)(.*)(?=\\[END\\])\",sen)\n",
        "    if len(finds) > 0:\n",
        "      sentences.append(finds[0].strip())\n",
        "  prompts.append({\n",
        "      \"prompt\": prompt,\n",
        "      \"sentences\": sentences\n",
        "  })\n",
        "  for sentence in sentences:\n",
        "    new_sentence_file.write(sentence+\"\\n\")\n",
        "\n",
        "new_sentence_file.close()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nud0L_JSGSF4",
        "outputId": "1bc702d7-a2d1-4917-f9f1-27dc0a669ec0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "67it [41:09, 36.86s/it]\n"
          ]
        }
      ]
    }
  ]
}