1_get_data_from_googlesheets.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "1_get_data_from_googlesheets.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jlBqPZ71Zx7B"
      },
      "source": [
        "# Processing a large Google Sheets file directly in Google Colab\n",
        "## and outputting it as an .xlsx file to Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8BrA8xOjnCf7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9d466c86-fcd8-46e9-c1c5-70a8fc83eb3b"
      },
      "source": [
        "# !pip install --upgrade gspread\n",
        "from google.colab import auth\n",
        "import pandas as pd\n",
        "pd.set_option('display.max_colwidth', 200)\n",
        "import gspread\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "auth.authenticate_user()\n",
        "\n",
        "gc = gspread.authorize(GoogleCredentials.get_application_default())\n",
        "\n",
        "worksheet = gc.open_by_url(\n",
        "    \"https://docs.google.com/spreadsheets/d/1pejcKIfD6wCvZOSVWO3E-PVe13411UYi0BFhsznbstA/edit?usp=sharing\"\n",
        ").worksheet(\"Archive\")\n",
        "\n",
        "# Convert to a DataFrame and render.\n",
        "\n",
        "df = pd.DataFrame(worksheet.get_all_records())\n",
        "print(df.columns)\n",
        "df = df.set_index(keys=\"id_str\")\n",
        "print('-- after index is set to \"id_str\"')\n",
        "print(df.columns)\n",
        "print(\"-- df.head():\")\n",
        "print(df.head(5))"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Index(['id_str', 'from_user', 'text', 'created_at', 'time', 'favorite_count',\n",
            "       'retweet_count', 'retweeted_status', 'lang', 'user_lang',\n",
            "       'user_location', 'user_followers_count', 'user_friends_count',\n",
            "       'in_reply_to_user_id_str', 'in_reply_to_screen_name',\n",
            "       'from_user_id_str', 'source', 'profile_image_url', 'status_url',\n",
            "       'entities_str'],\n",
            "      dtype='object')\n",
            "-- after index is set to \"id_str\"\n",
            "Index(['from_user', 'text', 'created_at', 'time', 'favorite_count',\n",
            "       'retweet_count', 'retweeted_status', 'lang', 'user_lang',\n",
            "       'user_location', 'user_followers_count', 'user_friends_count',\n",
            "       'in_reply_to_user_id_str', 'in_reply_to_screen_name',\n",
            "       'from_user_id_str', 'source', 'profile_image_url', 'status_url',\n",
            "       'entities_str'],\n",
            "      dtype='object')\n",
            "-- df.head():\n",
            "                          from_user  ...                                                                                                                                                                                             entities_str\n",
            "id_str                               ...                                                                                                                                                                                                         \n",
            "1315546075579789312    AregSerobyan  ...  {\"hashtags\":[{\"text\":\"Armenians\",\"indices\":[45,55]},{\"text\":\"Turkish\",\"indices\":[93,101]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"CivilNetTV\",\"name\":\"CIVILNET\",\"id\":59063969,\"id_str\":\"59063...\n",
            "1315546072715071490         GOVARIK  ...  {\"hashtags\":[{\"text\":\"Aliyev\",\"indices\":[19,26]},{\"text\":\"Christian\",\"indices\":[38,48]},{\"text\":\"Armenians\",\"indices\":[49,59]},{\"text\":\"Artsakh\",\"indices\":[90,98]}],\"symbols\":[],\"user_mentions\":[{...\n",
            "1315546059804946432         GOVARIK  ...  {\"hashtags\":[{\"text\":\"Azerbaijan\",\"indices\":[87,98]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"Karabakh_MoD\",\"name\":\"Artsakh Defense Army\",\"id\":3283108812,\"id_str\":\"3283108812\",\"indices\":[3,1...\n",
            "1315546059670720512  FrechouMartine  ...                                    {\"hashtags\":[],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"valerieboyer13\",\"name\":\"Valérie Boyer\",\"id\":276934698,\"id_str\":\"276934698\",\"indices\":[3,18]}],\"urls\":[]}\n",
            "1315546055187013632      TERTam_arm  ...  {\"hashtags\":[{\"text\":\"Armenia\",\"indices\":[103,111]},{\"text\":\"ArtsakhStrong\",\"indices\":[112,126]},{\"text\":\"Artsakh\",\"indices\":[127,135]},{\"text\":\"nagornokarabakh\",\"indices\":[136,152]},{\"text\":\"Stop...\n",
            "\n",
            "[5 rows x 19 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BN_FD5CEs2YU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d992dc74-3cbb-4734-96ed-472fcf215f61"
      },
      "source": [
        "print(\"-- df.info:\")\n",
        "df.info()\n",
        "# print(df.shape, '\\nRows total:', len(df.index), '\\nColumns total:', len(df.columns))\n",
        "print(\"\\n-- df.sample(n=4, random_state=1):\")\n",
        "df.sample(n=4, random_state=1)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-- df.info:\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Index: 189618 entries, 1315546075579789312 to 1313559695131541504\n",
            "Data columns (total 19 columns):\n",
            " #   Column                   Non-Null Count   Dtype \n",
            "---  ------                   --------------   ----- \n",
            " 0   from_user                189618 non-null  object\n",
            " 1   text                     189618 non-null  object\n",
            " 2   created_at               189618 non-null  object\n",
            " 3   time                     189618 non-null  object\n",
            " 4   favorite_count           189618 non-null  object\n",
            " 5   retweet_count            189618 non-null  object\n",
            " 6   retweeted_status         189618 non-null  object\n",
            " 7   lang                     189618 non-null  object\n",
            " 8   user_lang                189618 non-null  object\n",
            " 9   user_location            189618 non-null  object\n",
            " 10  user_followers_count     189618 non-null  object\n",
            " 11  user_friends_count       189618 non-null  object\n",
            " 12  in_reply_to_user_id_str  189618 non-null  object\n",
            " 13  in_reply_to_screen_name  189618 non-null  object\n",
            " 14  from_user_id_str         189618 non-null  object\n",
            " 15  source                   189618 non-null  object\n",
            " 16  profile_image_url        189618 non-null  object\n",
            " 17  status_url               189618 non-null  object\n",
            " 18  entities_str             189618 non-null  object\n",
            "dtypes: object(19)\n",
            "memory usage: 28.9+ MB\n",
            "\n",
            "-- df.sample(n=4, random_state=1):\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>from_user</th>\n",
              "      <th>text</th>\n",
              "      <th>created_at</th>\n",
              "      <th>time</th>\n",
              "      <th>favorite_count</th>\n",
              "      <th>retweet_count</th>\n",
              "      <th>retweeted_status</th>\n",
              "      <th>lang</th>\n",
              "      <th>user_lang</th>\n",
              "      <th>user_location</th>\n",
              "      <th>user_followers_count</th>\n",
              "      <th>user_friends_count</th>\n",
              "      <th>in_reply_to_user_id_str</th>\n",
              "      <th>in_reply_to_screen_name</th>\n",
              "      <th>from_user_id_str</th>\n",
              "      <th>source</th>\n",
              "      <th>profile_image_url</th>\n",
              "      <th>status_url</th>\n",
              "      <th>entities_str</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id_str</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1313940322368585728</th>\n",
              "      <td>sencani_besdi</td>\n",
              "      <td>#KharabaghisAzerbaijan  #StopArmenianOccupation #LongLiveAzerbaijan #LongLiveAzerbaijanArmy #StopArmenianAggression #ArmeniaShellingCivilians #NKpeace #Karabakh #KarabakhNow #Azerbaijan #Ganja #tu...</td>\n",
              "      <td>Wed Oct 07 20:32:39 +0000 2020</td>\n",
              "      <td>07/10/2020 21:32:39</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td></td>\n",
              "      <td>und</td>\n",
              "      <td></td>\n",
              "      <td>Baku</td>\n",
              "      <td>1549</td>\n",
              "      <td>209</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>737734328432660480</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/android\" rel=\"nofollow\"&gt;Twitter for Android&lt;/a&gt;</td>\n",
              "      <td>http://pbs.twimg.com/profile_images/1309063377168236548/lbFeZK6Q_normal.jpg</td>\n",
              "      <td>http://twitter.com/sencani_besdi/statuses/1313940322368585728</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"KharabaghisAzerbaijan\",\"indices\":[0,22]},{\"text\":\"StopArmenianOccupation\",\"indices\":[24,47]},{\"text\":\"LongLiveAzerbaijan\",\"indices\":[48,67]},{\"text\":\"LongLiveAzerbaijanArmy\",...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1313850138406850561</th>\n",
              "      <td>barikyang</td>\n",
              "      <td>RT @TERTam_arm: Our soldiers lead heroic battle for every inch of land - Arayik Harutyunyan\\n#Armenia #ArtsakhStrong #Artsakh #nagornokarabakh #StopAzerbaijaniAggression \\n\\nhttps://t.co/RsHVLMiYr...</td>\n",
              "      <td>Wed Oct 07 14:34:17 +0000 2020</td>\n",
              "      <td>07/10/2020 15:34:17</td>\n",
              "      <td></td>\n",
              "      <td>2</td>\n",
              "      <td>{extended_entities={media=[Ljava.lang.Object;@72d5ed2a}, metadata={result_type=recent, iso_language_code=en}, in_reply_to_status_id_str=null, in_reply_to_status_id=null, created_at=Wed Oct 07 14:2...</td>\n",
              "      <td>en</td>\n",
              "      <td></td>\n",
              "      <td>Armenia</td>\n",
              "      <td>36</td>\n",
              "      <td>405</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>2898148904</td>\n",
              "      <td>&lt;a href=\"https://mobile.twitter.com\" rel=\"nofollow\"&gt;Twitter Web App&lt;/a&gt;</td>\n",
              "      <td>http://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png</td>\n",
              "      <td>http://twitter.com/barikyang/statuses/1313850138406850561</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Armenia\",\"indices\":[92,100]},{\"text\":\"ArtsakhStrong\",\"indices\":[101,115]},{\"text\":\"Artsakh\",\"indices\":[116,124]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"TERTam_arm\",\"n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1315241548347568130</th>\n",
              "      <td>GevorgyanSevan</td>\n",
              "      <td>RT @naghdalyan: The cultural cradle of #Artsakh, the city of #Shushi, sees its Art &amp;amp; Youth Center severely damaged, due to deliberate shelling of civilian infrastructure by #Azerbaijan'i armed...</td>\n",
              "      <td>Sun Oct 11 10:43:15 +0000 2020</td>\n",
              "      <td>11/10/2020 11:43:15</td>\n",
              "      <td></td>\n",
              "      <td>243</td>\n",
              "      <td>{extended_entities={media=[Ljava.lang.Object;@9476ba0}, metadata={result_type=recent, iso_language_code=en}, in_reply_to_status_id_str=null, in_reply_to_status_id=null, created_at=Sun Oct 11 08:39...</td>\n",
              "      <td>en</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>49</td>\n",
              "      <td>587</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>1131417831969632256</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/android\" rel=\"nofollow\"&gt;Twitter for Android&lt;/a&gt;</td>\n",
              "      <td>http://pbs.twimg.com/profile_images/1256942080590524416/epeFhrlj_normal.jpg</td>\n",
              "      <td>http://twitter.com/GevorgyanSevan/statuses/1315241548347568130</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Artsakh\",\"indices\":[39,47]},{\"text\":\"Shushi\",\"indices\":[61,68]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"naghdalyan\",\"name\":\"Anna A. Naghdalyan\",\"id\":133765231,\"id_str\"...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1314310312527446016</th>\n",
              "      <td>MariaSosyan</td>\n",
              "      <td>RT @ArtyomTonoyan: This photo of the Ghazanchetsots Cathedral I took the last time I visited Shushi in 2015 #NagornoKarabagh #Karabakh https://t.co/FbqmjuBhyO https://t.co/xJsfqTAcyF</td>\n",
              "      <td>Thu Oct 08 21:02:51 +0000 2020</td>\n",
              "      <td>08/10/2020 22:02:51</td>\n",
              "      <td></td>\n",
              "      <td>24</td>\n",
              "      <td>{extended_entities={media=[Ljava.lang.Object;@6deb374c}, metadata={result_type=recent, iso_language_code=en}, quoted_status={extended_entities={media=[Ljava.lang.Object;@440be595}, metadata={resul...</td>\n",
              "      <td>en</td>\n",
              "      <td></td>\n",
              "      <td>California, USA</td>\n",
              "      <td>2434</td>\n",
              "      <td>591</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>2202162126</td>\n",
              "      <td>&lt;a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\"&gt;Twitter for iPhone&lt;/a&gt;</td>\n",
              "      <td>http://pbs.twimg.com/profile_images/1139684155262767105/6NWM5Iyf_normal.jpg</td>\n",
              "      <td>http://twitter.com/MariaSosyan/statuses/1314310312527446016</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"NagornoKarabagh\",\"indices\":[108,124]},{\"text\":\"Karabakh\",\"indices\":[125,134]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"ArtyomTonoyan\",\"name\":\"Dr. Artyom Tonoyan\",\"id\":3...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                          from_user  ...                                                                                                                                                                                             entities_str\n",
              "id_str                               ...                                                                                                                                                                                                         \n",
              "1313940322368585728   sencani_besdi  ...  {\"hashtags\":[{\"text\":\"KharabaghisAzerbaijan\",\"indices\":[0,22]},{\"text\":\"StopArmenianOccupation\",\"indices\":[24,47]},{\"text\":\"LongLiveAzerbaijan\",\"indices\":[48,67]},{\"text\":\"LongLiveAzerbaijanArmy\",...\n",
              "1313850138406850561       barikyang  ...  {\"hashtags\":[{\"text\":\"Armenia\",\"indices\":[92,100]},{\"text\":\"ArtsakhStrong\",\"indices\":[101,115]},{\"text\":\"Artsakh\",\"indices\":[116,124]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"TERTam_arm\",\"n...\n",
              "1315241548347568130  GevorgyanSevan  ...  {\"hashtags\":[{\"text\":\"Artsakh\",\"indices\":[39,47]},{\"text\":\"Shushi\",\"indices\":[61,68]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"naghdalyan\",\"name\":\"Anna A. Naghdalyan\",\"id\":133765231,\"id_str\"...\n",
              "1314310312527446016     MariaSosyan  ...  {\"hashtags\":[{\"text\":\"NagornoKarabagh\",\"indices\":[108,124]},{\"text\":\"Karabakh\",\"indices\":[125,134]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"ArtyomTonoyan\",\"name\":\"Dr. Artyom Tonoyan\",\"id\":3...\n",
              "\n",
              "[4 rows x 19 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mGVEgi98phsj"
      },
      "source": [
        "# DROP irrelevant columns\n",
        "df = df.drop(\n",
        "    columns=[\n",
        "        \"user_lang\",\n",
        "        \"in_reply_to_user_id_str\",\n",
        "        \"in_reply_to_screen_name\",\n",
        "        \"from_user_id_str\",\n",
        "        \"source\",\n",
        "        \"profile_image_url\",\n",
        "        \"status_url\",\n",
        "    ]\n",
        ")\n",
        "# print(df.shape, '\\nRows total:', len(df.index), '\\nColumns total:', len(df.columns))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ighl4PHSuuA9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "a739eed7-0b2f-4682-e7f1-6227423f9de5"
      },
      "source": [
        "print(\"-- df.info:\")\n",
        "df.info()\n",
        "# Print a sample of n rows\n",
        "print(\"\\n-- df.sample(n=4, random_state=1):\")\n",
        "df.sample(n=4, random_state=1)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-- df.info:\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Index: 189618 entries, 1315546075579789312 to 1313559695131541504\n",
            "Data columns (total 12 columns):\n",
            " #   Column                Non-Null Count   Dtype \n",
            "---  ------                --------------   ----- \n",
            " 0   from_user             189618 non-null  object\n",
            " 1   text                  189618 non-null  object\n",
            " 2   created_at            189618 non-null  object\n",
            " 3   time                  189618 non-null  object\n",
            " 4   favorite_count        189618 non-null  object\n",
            " 5   retweet_count         189618 non-null  object\n",
            " 6   retweeted_status      189618 non-null  object\n",
            " 7   lang                  189618 non-null  object\n",
            " 8   user_location         189618 non-null  object\n",
            " 9   user_followers_count  189618 non-null  object\n",
            " 10  user_friends_count    189618 non-null  object\n",
            " 11  entities_str          189618 non-null  object\n",
            "dtypes: object(12)\n",
            "memory usage: 18.8+ MB\n",
            "\n",
            "-- df.sample(n=4, random_state=1):\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>from_user</th>\n",
              "      <th>text</th>\n",
              "      <th>created_at</th>\n",
              "      <th>time</th>\n",
              "      <th>favorite_count</th>\n",
              "      <th>retweet_count</th>\n",
              "      <th>retweeted_status</th>\n",
              "      <th>lang</th>\n",
              "      <th>user_location</th>\n",
              "      <th>user_followers_count</th>\n",
              "      <th>user_friends_count</th>\n",
              "      <th>entities_str</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id_str</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1313940322368585728</th>\n",
              "      <td>sencani_besdi</td>\n",
              "      <td>#KharabaghisAzerbaijan  #StopArmenianOccupation #LongLiveAzerbaijan #LongLiveAzerbaijanArmy #StopArmenianAggression #ArmeniaShellingCivilians #NKpeace #Karabakh #KarabakhNow #Azerbaijan #Ganja #tu...</td>\n",
              "      <td>Wed Oct 07 20:32:39 +0000 2020</td>\n",
              "      <td>07/10/2020 21:32:39</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td></td>\n",
              "      <td>und</td>\n",
              "      <td>Baku</td>\n",
              "      <td>1549</td>\n",
              "      <td>209</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"KharabaghisAzerbaijan\",\"indices\":[0,22]},{\"text\":\"StopArmenianOccupation\",\"indices\":[24,47]},{\"text\":\"LongLiveAzerbaijan\",\"indices\":[48,67]},{\"text\":\"LongLiveAzerbaijanArmy\",...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1313850138406850561</th>\n",
              "      <td>barikyang</td>\n",
              "      <td>RT @TERTam_arm: Our soldiers lead heroic battle for every inch of land - Arayik Harutyunyan\\n#Armenia #ArtsakhStrong #Artsakh #nagornokarabakh #StopAzerbaijaniAggression \\n\\nhttps://t.co/RsHVLMiYr...</td>\n",
              "      <td>Wed Oct 07 14:34:17 +0000 2020</td>\n",
              "      <td>07/10/2020 15:34:17</td>\n",
              "      <td></td>\n",
              "      <td>2</td>\n",
              "      <td>{extended_entities={media=[Ljava.lang.Object;@72d5ed2a}, metadata={result_type=recent, iso_language_code=en}, in_reply_to_status_id_str=null, in_reply_to_status_id=null, created_at=Wed Oct 07 14:2...</td>\n",
              "      <td>en</td>\n",
              "      <td>Armenia</td>\n",
              "      <td>36</td>\n",
              "      <td>405</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Armenia\",\"indices\":[92,100]},{\"text\":\"ArtsakhStrong\",\"indices\":[101,115]},{\"text\":\"Artsakh\",\"indices\":[116,124]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"TERTam_arm\",\"n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1315241548347568130</th>\n",
              "      <td>GevorgyanSevan</td>\n",
              "      <td>RT @naghdalyan: The cultural cradle of #Artsakh, the city of #Shushi, sees its Art &amp;amp; Youth Center severely damaged, due to deliberate shelling of civilian infrastructure by #Azerbaijan'i armed...</td>\n",
              "      <td>Sun Oct 11 10:43:15 +0000 2020</td>\n",
              "      <td>11/10/2020 11:43:15</td>\n",
              "      <td></td>\n",
              "      <td>243</td>\n",
              "      <td>{extended_entities={media=[Ljava.lang.Object;@9476ba0}, metadata={result_type=recent, iso_language_code=en}, in_reply_to_status_id_str=null, in_reply_to_status_id=null, created_at=Sun Oct 11 08:39...</td>\n",
              "      <td>en</td>\n",
              "      <td></td>\n",
              "      <td>49</td>\n",
              "      <td>587</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Artsakh\",\"indices\":[39,47]},{\"text\":\"Shushi\",\"indices\":[61,68]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"naghdalyan\",\"name\":\"Anna A. Naghdalyan\",\"id\":133765231,\"id_str\"...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1314310312527446016</th>\n",
              "      <td>MariaSosyan</td>\n",
              "      <td>RT @ArtyomTonoyan: This photo of the Ghazanchetsots Cathedral I took the last time I visited Shushi in 2015 #NagornoKarabagh #Karabakh https://t.co/FbqmjuBhyO https://t.co/xJsfqTAcyF</td>\n",
              "      <td>Thu Oct 08 21:02:51 +0000 2020</td>\n",
              "      <td>08/10/2020 22:02:51</td>\n",
              "      <td></td>\n",
              "      <td>24</td>\n",
              "      <td>{extended_entities={media=[Ljava.lang.Object;@6deb374c}, metadata={result_type=recent, iso_language_code=en}, quoted_status={extended_entities={media=[Ljava.lang.Object;@440be595}, metadata={resul...</td>\n",
              "      <td>en</td>\n",
              "      <td>California, USA</td>\n",
              "      <td>2434</td>\n",
              "      <td>591</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"NagornoKarabagh\",\"indices\":[108,124]},{\"text\":\"Karabakh\",\"indices\":[125,134]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"ArtyomTonoyan\",\"name\":\"Dr. Artyom Tonoyan\",\"id\":3...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                          from_user  ...                                                                                                                                                                                             entities_str\n",
              "id_str                               ...                                                                                                                                                                                                         \n",
              "1313940322368585728   sencani_besdi  ...  {\"hashtags\":[{\"text\":\"KharabaghisAzerbaijan\",\"indices\":[0,22]},{\"text\":\"StopArmenianOccupation\",\"indices\":[24,47]},{\"text\":\"LongLiveAzerbaijan\",\"indices\":[48,67]},{\"text\":\"LongLiveAzerbaijanArmy\",...\n",
              "1313850138406850561       barikyang  ...  {\"hashtags\":[{\"text\":\"Armenia\",\"indices\":[92,100]},{\"text\":\"ArtsakhStrong\",\"indices\":[101,115]},{\"text\":\"Artsakh\",\"indices\":[116,124]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"TERTam_arm\",\"n...\n",
              "1315241548347568130  GevorgyanSevan  ...  {\"hashtags\":[{\"text\":\"Artsakh\",\"indices\":[39,47]},{\"text\":\"Shushi\",\"indices\":[61,68]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"naghdalyan\",\"name\":\"Anna A. Naghdalyan\",\"id\":133765231,\"id_str\"...\n",
              "1314310312527446016     MariaSosyan  ...  {\"hashtags\":[{\"text\":\"NagornoKarabagh\",\"indices\":[108,124]},{\"text\":\"Karabakh\",\"indices\":[125,134]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"ArtyomTonoyan\",\"name\":\"Dr. Artyom Tonoyan\",\"id\":3...\n",
              "\n",
              "[4 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sBz9YzbBo0Aw",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "60a15dd0-85a3-4c27-edba-72999d71857a"
      },
      "source": [
        "# Selecting duplicate rows except first\n",
        "# occurrence based on all columns\n",
        "duplicate = df[df.duplicated()]\n",
        "print(\"Duplicate Rows :\")\n",
        "# Print the resultant Dataframe\n",
        "display(duplicate)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Duplicate Rows :\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>from_user</th>\n",
              "      <th>text</th>\n",
              "      <th>created_at</th>\n",
              "      <th>time</th>\n",
              "      <th>favorite_count</th>\n",
              "      <th>retweet_count</th>\n",
              "      <th>retweeted_status</th>\n",
              "      <th>lang</th>\n",
              "      <th>user_location</th>\n",
              "      <th>user_followers_count</th>\n",
              "      <th>user_friends_count</th>\n",
              "      <th>entities_str</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id_str</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1315541732184989696</th>\n",
              "      <td>GOVARIK</td>\n",
              "      <td>The #world should recognize #Artsakh's people right to self-determination! We will stand for ourselves, we will raise our voices! The world should stop injustice, aggression, and terrorism! #Recog...</td>\n",
              "      <td>Mon Oct 12 06:36:05 +0000 2020</td>\n",
              "      <td>12/10/2020 07:36:05</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td></td>\n",
              "      <td>en</td>\n",
              "      <td>Armenia</td>\n",
              "      <td>91</td>\n",
              "      <td>475</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"world\",\"indices\":[4,10]},{\"text\":\"Artsakh\",\"indices\":[28,36]},{\"text\":\"RecognizeArtsakh\",\"indices\":[190,207]},{\"text\":\"StopAzerbaijaniAggression\",\"indices\":[208,234]}],\"symbo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1315537953326858242</th>\n",
              "      <td>LragirAm</td>\n",
              "      <td>Լոս Անջելեսի հայությունը՝ #Արցախի թիկունքին https://t.co/MYGP20yprZ\\n\\n#Artsakh #ArtsakhStrong #LosAngeles #StopAzerbaijaniAggression</td>\n",
              "      <td>Mon Oct 12 06:21:04 +0000 2020</td>\n",
              "      <td>12/10/2020 07:21:04</td>\n",
              "      <td>2</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>hy</td>\n",
              "      <td>Yerevan, Armenia</td>\n",
              "      <td>4495</td>\n",
              "      <td>298</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Արցախի\",\"indices\":[26,33]},{\"text\":\"Artsakh\",\"indices\":[69,77]},{\"text\":\"ArtsakhStrong\",\"indices\":[78,92]},{\"text\":\"LosAngeles\",\"indices\":[93,104]},{\"text\":\"StopAzerbaijaniAg...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1315534628384468992</th>\n",
              "      <td>garenovsepyan</td>\n",
              "      <td>@CBSLA running low on your ad campaign$$$? FACT CHECK BEFORE YOU GET SOMEONE ON AIR #ArtsakhIsArmenia #Armenia #ArtsakhStong #StopAliyev #StopTurkey #SanctionAzerbaijan  #Karabakh #Artsakh #Recogn...</td>\n",
              "      <td>Mon Oct 12 06:07:51 +0000 2020</td>\n",
              "      <td>12/10/2020 07:07:51</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td></td>\n",
              "      <td>en</td>\n",
              "      <td>Los Angeles, CA</td>\n",
              "      <td>240</td>\n",
              "      <td>232</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"ArtsakhIsArmenia\",\"indices\":[84,101]},{\"text\":\"Armenia\",\"indices\":[102,110]},{\"text\":\"ArtsakhStong\",\"indices\":[111,124]},{\"text\":\"StopAliyev\",\"indices\":[125,136]},{\"text\":\"St...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1315525442347110400</th>\n",
              "      <td>KarenMAzaryan</td>\n",
              "      <td>ATUALIZAÇÃO DIA 16: Na guerra deliberadamente lançada pelo #Azerbaijão em 27 de setembro de 2020 com o apoio da #turquia contra #Artsakh / # NagornoKarabakh que se espalhou pela #Armenia e afetou ...</td>\n",
              "      <td>Mon Oct 12 05:31:21 +0000 2020</td>\n",
              "      <td>12/10/2020 06:31:21</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>pt</td>\n",
              "      <td>Yerevan, Armenia</td>\n",
              "      <td>89</td>\n",
              "      <td>393</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Azerbaijão\",\"indices\":[59,70]},{\"text\":\"turquia\",\"indices\":[112,120]},{\"text\":\"Artsakh\",\"indices\":[128,136]},{\"text\":\"Armenia\",\"indices\":[178,186]},{\"text\":\"Irao\",\"indices\":[...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1315522509500936193</th>\n",
              "      <td>armen_zee</td>\n",
              "      <td>Account made 04/2020, 50k followers, ONLY tweets are about Armenia and Azerbaijan. clear as daylight this “attack on Azerbaijan citizens” is #FakeNews wouldn’t need bots to spread the truth. #Arme...</td>\n",
              "      <td>Mon Oct 12 05:19:42 +0000 2020</td>\n",
              "      <td>12/10/2020 06:19:42</td>\n",
              "      <td>1</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>en</td>\n",
              "      <td></td>\n",
              "      <td>213</td>\n",
              "      <td>350</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"FakeNews\",\"indices\":[141,150]},{\"text\":\"ArmeniaWantsPeace\",\"indices\":[191,209]},{\"text\":\"Armenia\",\"indices\":[210,218]},{\"text\":\"ArstakhStrong\",\"indices\":[219,233]},{\"text\":\"A...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1313576366030364672</th>\n",
              "      <td>Aysel_Aliyeva1</td>\n",
              "      <td>@houshigk Lol there is not artsakh there is #Karabakh teritory of AZERBAİJAN BTW #Azerbajian also send help financial help TO #Lebanon</td>\n",
              "      <td>Tue Oct 06 20:26:25 +0000 2020</td>\n",
              "      <td>06/10/2020 21:26:25</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>en</td>\n",
              "      <td>Dubai, United Arab Emirates</td>\n",
              "      <td>1096</td>\n",
              "      <td>373</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Karabakh\",\"indices\":[44,53]},{\"text\":\"Azerbajian\",\"indices\":[81,92]},{\"text\":\"Lebanon\",\"indices\":[126,134]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"houshigk\",\"name\":\"H...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1313575955168817154</th>\n",
              "      <td>VroApar</td>\n",
              "      <td>#Armenia #Artsakh #ArmeniaStrong #ArtsakhSrtong #LosAngeles https://t.co/QuYAVyxli2</td>\n",
              "      <td>Tue Oct 06 20:24:47 +0000 2020</td>\n",
              "      <td>06/10/2020 21:24:47</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>und</td>\n",
              "      <td></td>\n",
              "      <td>24</td>\n",
              "      <td>81</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Armenia\",\"indices\":[0,8]},{\"text\":\"Artsakh\",\"indices\":[9,17]},{\"text\":\"ArmeniaStrong\",\"indices\":[18,32]},{\"text\":\"ArtsakhSrtong\",\"indices\":[33,47]},{\"text\":\"LosAngeles\",\"indi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1313574360939147266</th>\n",
              "      <td>anahit_ohanyann</td>\n",
              "      <td>#StopAzerbaijaniAggression #StopErdogan \\n#AzerbaijaniAggression #NKpeace #ArtsakhStrong #WarInKarabakh #StopAliyev #Karabakh #Armenia #Azerbaijan #Turkey #StopAzerbaijaniAggression https://t.co/O...</td>\n",
              "      <td>Tue Oct 06 20:18:27 +0000 2020</td>\n",
              "      <td>06/10/2020 21:18:27</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td></td>\n",
              "      <td>und</td>\n",
              "      <td>Москва, Россия</td>\n",
              "      <td>610</td>\n",
              "      <td>694</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"StopAzerbaijaniAggression\",\"indices\":[0,26]},{\"text\":\"StopErdogan\",\"indices\":[27,39]},{\"text\":\"AzerbaijaniAggression\",\"indices\":[41,63]},{\"text\":\"NKpeace\",\"indices\":[64,72]},...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1313564593927708679</th>\n",
              "      <td>ArmenOrujyan</td>\n",
              "      <td>Cowards! #Erdogan's cronies, the #Azeries shelling the civilian population of the capital of #NagornoKarabakh / #Artsakh. Those of you quiet, waiting on the sidelines, you know who you are, what g...</td>\n",
              "      <td>Tue Oct 06 19:39:38 +0000 2020</td>\n",
              "      <td>06/10/2020 20:39:38</td>\n",
              "      <td>1</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>en</td>\n",
              "      <td></td>\n",
              "      <td>1790</td>\n",
              "      <td>58</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Erdogan\",\"indices\":[9,17]},{\"text\":\"Azeries\",\"indices\":[33,41]},{\"text\":\"NagornoKarabakh\",\"indices\":[93,109]},{\"text\":\"Artsakh\",\"indices\":[112,120]}],\"symbols\":[],\"user_menti...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1313560823558737921</th>\n",
              "      <td>PaulEFishr</td>\n",
              "      <td>There is no need for #Pashinyan to undermine perfectly legitimate arguments for the right to self-determination of Armenians in #Karabakh by failing to acknowledge these facts.</td>\n",
              "      <td>Tue Oct 06 19:24:39 +0000 2020</td>\n",
              "      <td>06/10/2020 20:24:39</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>en</td>\n",
              "      <td></td>\n",
              "      <td>904</td>\n",
              "      <td>1590</td>\n",
              "      <td>{\"hashtags\":[{\"text\":\"Pashinyan\",\"indices\":[21,31]},{\"text\":\"Karabakh\",\"indices\":[128,137]}],\"symbols\":[],\"user_mentions\":[],\"urls\":[]}</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2465 rows × 12 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                           from_user  ...                                                                                                                                                                                             entities_str\n",
              "id_str                                ...                                                                                                                                                                                                         \n",
              "1315541732184989696          GOVARIK  ...  {\"hashtags\":[{\"text\":\"world\",\"indices\":[4,10]},{\"text\":\"Artsakh\",\"indices\":[28,36]},{\"text\":\"RecognizeArtsakh\",\"indices\":[190,207]},{\"text\":\"StopAzerbaijaniAggression\",\"indices\":[208,234]}],\"symbo...\n",
              "1315537953326858242         LragirAm  ...  {\"hashtags\":[{\"text\":\"Արցախի\",\"indices\":[26,33]},{\"text\":\"Artsakh\",\"indices\":[69,77]},{\"text\":\"ArtsakhStrong\",\"indices\":[78,92]},{\"text\":\"LosAngeles\",\"indices\":[93,104]},{\"text\":\"StopAzerbaijaniAg...\n",
              "1315534628384468992    garenovsepyan  ...  {\"hashtags\":[{\"text\":\"ArtsakhIsArmenia\",\"indices\":[84,101]},{\"text\":\"Armenia\",\"indices\":[102,110]},{\"text\":\"ArtsakhStong\",\"indices\":[111,124]},{\"text\":\"StopAliyev\",\"indices\":[125,136]},{\"text\":\"St...\n",
              "1315525442347110400    KarenMAzaryan  ...  {\"hashtags\":[{\"text\":\"Azerbaijão\",\"indices\":[59,70]},{\"text\":\"turquia\",\"indices\":[112,120]},{\"text\":\"Artsakh\",\"indices\":[128,136]},{\"text\":\"Armenia\",\"indices\":[178,186]},{\"text\":\"Irao\",\"indices\":[...\n",
              "1315522509500936193        armen_zee  ...  {\"hashtags\":[{\"text\":\"FakeNews\",\"indices\":[141,150]},{\"text\":\"ArmeniaWantsPeace\",\"indices\":[191,209]},{\"text\":\"Armenia\",\"indices\":[210,218]},{\"text\":\"ArstakhStrong\",\"indices\":[219,233]},{\"text\":\"A...\n",
              "...                              ...  ...                                                                                                                                                                                                      ...\n",
              "1313576366030364672   Aysel_Aliyeva1  ...  {\"hashtags\":[{\"text\":\"Karabakh\",\"indices\":[44,53]},{\"text\":\"Azerbajian\",\"indices\":[81,92]},{\"text\":\"Lebanon\",\"indices\":[126,134]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"houshigk\",\"name\":\"H...\n",
              "1313575955168817154          VroApar  ...  {\"hashtags\":[{\"text\":\"Armenia\",\"indices\":[0,8]},{\"text\":\"Artsakh\",\"indices\":[9,17]},{\"text\":\"ArmeniaStrong\",\"indices\":[18,32]},{\"text\":\"ArtsakhSrtong\",\"indices\":[33,47]},{\"text\":\"LosAngeles\",\"indi...\n",
              "1313574360939147266  anahit_ohanyann  ...  {\"hashtags\":[{\"text\":\"StopAzerbaijaniAggression\",\"indices\":[0,26]},{\"text\":\"StopErdogan\",\"indices\":[27,39]},{\"text\":\"AzerbaijaniAggression\",\"indices\":[41,63]},{\"text\":\"NKpeace\",\"indices\":[64,72]},...\n",
              "1313564593927708679     ArmenOrujyan  ...  {\"hashtags\":[{\"text\":\"Erdogan\",\"indices\":[9,17]},{\"text\":\"Azeries\",\"indices\":[33,41]},{\"text\":\"NagornoKarabakh\",\"indices\":[93,109]},{\"text\":\"Artsakh\",\"indices\":[112,120]}],\"symbols\":[],\"user_menti...\n",
              "1313560823558737921       PaulEFishr  ...                                                                  {\"hashtags\":[{\"text\":\"Pashinyan\",\"indices\":[21,31]},{\"text\":\"Karabakh\",\"indices\":[128,137]}],\"symbols\":[],\"user_mentions\":[],\"urls\":[]}\n",
              "\n",
              "[2465 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JM1eoWFm4tbF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "54dc4316-79a1-43bb-e8c0-9e42bf6be419"
      },
      "source": [
        "# Selecting duplicate rows except first\n",
        "# occurrence based on all columns\n",
        "df = df.drop_duplicates()\n",
        "print(\"-- df.info:\")\n",
        "df.info()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-- df.info:\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Index: 187153 entries, 1315546075579789312 to 1313559695131541504\n",
            "Data columns (total 12 columns):\n",
            " #   Column                Non-Null Count   Dtype \n",
            "---  ------                --------------   ----- \n",
            " 0   from_user             187153 non-null  object\n",
            " 1   text                  187153 non-null  object\n",
            " 2   created_at            187153 non-null  object\n",
            " 3   time                  187153 non-null  object\n",
            " 4   favorite_count        187153 non-null  object\n",
            " 5   retweet_count         187153 non-null  object\n",
            " 6   retweeted_status      187153 non-null  object\n",
            " 7   lang                  187153 non-null  object\n",
            " 8   user_location         187153 non-null  object\n",
            " 9   user_followers_count  187153 non-null  object\n",
            " 10  user_friends_count    187153 non-null  object\n",
            " 11  entities_str          187153 non-null  object\n",
            "dtypes: object(12)\n",
            "memory usage: 18.6+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EMWQLNRB3FbM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c15f7778-724e-4989-aba0-8dc319db4657"
      },
      "source": [
        "# Drop the rows where all elements are missing.\n",
        "df = df.dropna(how=\"all\")\n",
        "print(\"-- df.info:\")\n",
        "df.info()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-- df.info:\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Index: 187153 entries, 1315546075579789312 to 1313559695131541504\n",
            "Data columns (total 12 columns):\n",
            " #   Column                Non-Null Count   Dtype \n",
            "---  ------                --------------   ----- \n",
            " 0   from_user             187153 non-null  object\n",
            " 1   text                  187153 non-null  object\n",
            " 2   created_at            187153 non-null  object\n",
            " 3   time                  187153 non-null  object\n",
            " 4   favorite_count        187153 non-null  object\n",
            " 5   retweet_count         187153 non-null  object\n",
            " 6   retweeted_status      187153 non-null  object\n",
            " 7   lang                  187153 non-null  object\n",
            " 8   user_location         187153 non-null  object\n",
            " 9   user_followers_count  187153 non-null  object\n",
            " 10  user_friends_count    187153 non-null  object\n",
            " 11  entities_str          187153 non-null  object\n",
            "dtypes: object(12)\n",
            "memory usage: 18.6+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BnPuq3uubtVp"
      },
      "source": [
        "### Save resulting dataframe to Google Drive as an .xlsx file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rAXJ8pUZ6c0s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ec4e5a7e-87b2-41c2-dbdc-ab258ceb468a"
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount(\"/content/drive\")\n",
        "\n",
        "# Write dataframe back to an XLSX file\n",
        "outputfile_path = \"/content/drive/My Drive/Social Media Analytics/data_no-duplicates.xlsx\"\n",
        "with pd.ExcelWriter(outputfile_path) as writer:\n",
        "    df.to_excel(writer, sheet_name=\"data\")\n",
        "    writer.save()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DAZyauIyeobU",
        "outputId": "38ccfaab-077c-46c5-d2e4-20ee4d28ab53"
      },
      "source": [
        "from os import stat\n",
        "print(f\"Resulting file:\\n {outputfile_path}\\n {round(os.stat(outputfile_path).st_size/(1024*1024), 2)}MB\")"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Resulting file:\n",
            " /content/drive/My Drive/Social Media Analytics/data_no-duplicates.xlsx\n",
            " 77.35MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fX9vB5Unf2X-"
      },
      "source": [
        "### Run next cell in order to download resulting file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lwdTFh7YdKn-"
      },
      "source": [
        "from google.colab import files\n",
        "files.download(outputfile_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BDSemnJt-wZY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "48145f71-afa0-4a86-a12c-3a4287dcc78e"
      },
      "source": [
        "# When done writing to Google Drive:\n",
        "drive.flush_and_unmount()\n",
        "print(\n",
        "    f\"All changes made in this colab session should now be visible in Drive at {outputfile_path}\"\n",
        ")"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All changes made in this colab session should now be visible in Drive at /content/drive/My Drive/Social Media Analytics/data_no-duplicates.xlsx\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}