notebooks/EDA_Python150k.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VgerLYy576ej"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "sns.set_style('darkgrid')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7TlByObr7KZV"
   },
   "source": [
    "# Basic statistics of preprocessed Python150k dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "R7aRqTf37KZX",
    "outputId": "c135c35d-d2dc-4305-a49b-07b508838857"
   },
   "outputs": [],
   "source": [
    "!unzip parsed.zip\n",
    "!ls parsed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "wQXjOjHb7KZe",
    "outputId": "8dfdc278-be95-4be0-8615-ae68e682adab"
   },
   "outputs": [],
   "source": [
    "prefix = \"parsed\"\n",
    "\n",
    "comments = open(f\"{prefix}/python150k_comments.txt\", \"rt\")\n",
    "comments = comments.readlines()\n",
    "print(\"Len(comments):\", len(comments))\n",
    "print(\"comments[10]:\", comments[10])\n",
    "\n",
    "docstrings = open(f\"{prefix}/python150k_docstrings.txt\", \"rt\")\n",
    "docstrings = docstrings.readlines()\n",
    "print(\"Len(docstrings):\", len(docstrings))\n",
    "print(\"docstrings[10]:\", docstrings[10])\n",
    "\n",
    "functions = open(f\"{prefix}/python150k_sequence.txt\", \"rt\")\n",
    "functions = functions.readlines()\n",
    "print(\"Len(functions):\", len(functions))\n",
    "print(\"functions[10]:\", functions[10])\n",
    "\n",
    "ast_functions = open(f\"{prefix}/python150k_ast.txt\", \"rt\")\n",
    "ast_functions = ast_functions.readlines()\n",
    "print(\"Len(ast_functions):\", len(ast_functions))\n",
    "print(\"ast_functions[10]:\", ast_functions[10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ny1jSdkY8Vfh"
   },
   "source": [
    "Train/Test split:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6fcfZTOG8XIG"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_comments, test_comments = train_test_split(comments, test_size=0.2)\n",
    "train_functions, test_functions = train_test_split(functions, test_size=0.2)\n",
    "train_ast_functions, test_ast_functions = train_test_split(ast_functions, test_size=0.2)\n",
    "train_docstrings, test_docstrings = train_test_split(docstrings, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QTtiE1Hz8-cV",
    "outputId": "1c7e2232-992c-4fbe-a63c-4e6c8ca40edf"
   },
   "outputs": [],
   "source": [
    "print(\"# functions in train:\", len(train_functions))\n",
    "print(\"# functions in test:\", len(test_functions))\n",
    "print(\"# comments in train:\", len(train_comments))\n",
    "print(\"# comments in test:\", len(test_comments))\n",
    "print(\"# docstrings in train:\", len(train_docstrings))\n",
    "print(\"# docstrings in test:\", len(test_docstrings))\n",
    "print(\"# ast functions in train:\", len(train_ast_functions))\n",
    "print(\"# ast functions in test:\", len(test_ast_functions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2q1zltuA9cCB"
   },
   "source": [
    "**Note:** <br>\n",
    "Processed files: **6854** <br>\n",
    "Canceled files:  **1459** <br>\n",
    "Total files: **8313**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "i7Q6vUR87KZf"
   },
   "source": [
    "# Distributions:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-Ufy6KfW_CpN"
   },
   "source": [
    "**Usual functions:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-X9Uxfha94mz"
   },
   "outputs": [],
   "source": [
    "# Count functions length histogram\n",
    "train_func_lengths = list(map(len, train_functions))\n",
    "test_func_lengths = list(map(len, test_functions))\n",
    "\n",
    "# Filter bad cases\n",
    "train_func_lengths = list(filter(lambda len: len < 1e4, train_func_lengths))\n",
    "test_func_lengths = list(filter(lambda len: len < 1e4, test_func_lengths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 624
    },
    "id": "zknHiDeF7KZf",
    "outputId": "627ac054-8cb9-42f2-d7a8-b2dcd334127b"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15, 8))\n",
    "plt.xlabel(\"Length of the function\", fontsize=20)\n",
    "plt.ylabel(\"Density\", fontsize=20)\n",
    "ax = sns.distplot(train_func_lengths, color='#007b7f', label='Functions from train set', kde_kws=dict(linewidth=3))\n",
    "ax.patch.set_edgecolor('black')  \n",
    "ax.patch.set_linewidth('2')\n",
    "ax2 = sns.distplot(test_func_lengths, color='g', label='Functions from test set', kde_kws=dict(linewidth=3))\n",
    "ax2.patch.set_edgecolor('black')  \n",
    "ax2.patch.set_linewidth('2')\n",
    "plt.setp(ax.patches, linewidth=3)\n",
    "plt.title(\"Functions lengths distributions\", fontsize=30)\n",
    "plt.xlim((-10, 2000))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mes6dX3w_GyH"
   },
   "source": [
    "**AST-processed functions:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "N_vbA_o9_BFg"
   },
   "outputs": [],
   "source": [
    "# Count functions length histogram\n",
    "train_func_lengths = list(map(len, train_ast_functions))\n",
    "test_func_lengths = list(map(len, test_ast_functions))\n",
    "\n",
    "# Filter bad cases\n",
    "train_func_lengths = list(filter(lambda len: len < 1e4, train_func_lengths))\n",
    "test_func_lengths = list(filter(lambda len: len < 1e4, test_func_lengths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 624
    },
    "id": "kf7cVysp_qi-",
    "outputId": "05cd6205-9fe9-4f8a-de6f-b8c35b4a1e75"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15, 8))\n",
    "plt.xlabel(\"Length of the function\", fontsize=20)\n",
    "plt.ylabel(\"Density\", fontsize=20)\n",
    "ax = sns.distplot(train_func_lengths, color='#007b7f', label='Functions from train set', kde_kws=dict(linewidth=3))\n",
    "ax.patch.set_edgecolor('black')  \n",
    "ax.patch.set_linewidth('2')\n",
    "ax2 = sns.distplot(test_func_lengths, color='g', label='Functions from test set', kde_kws=dict(linewidth=3))\n",
    "ax2.patch.set_edgecolor('black')  \n",
    "ax2.patch.set_linewidth('2')\n",
    "plt.setp(ax.patches, linewidth=3)\n",
    "plt.title(\"AST processed functions lengths distributions\", fontsize=30)\n",
    "plt.xlim((-10, 2000))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eq6iT3sJAMns"
   },
   "source": [
    "# Summary lengths:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3y89WZxFANGB",
    "outputId": "75367442-c949-4c9d-fcca-3becfb8de7aa"
   },
   "outputs": [],
   "source": [
    "train_lengths_comments = list(map(len, train_comments))\n",
    "test_lengths_comments = list(map(len, test_comments))\n",
    "\n",
    "print(\"Mean length of train comments in SYMBOLS:\", np.array(train_lengths_comments).mean())\n",
    "print(\"Mean length of test comments in SYMBOLS:\", np.array(test_lengths_comments).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "q5dCFaxwA_6p",
    "outputId": "3044c2b9-5761-4571-d7a7-21a0d3746b0d"
   },
   "outputs": [],
   "source": [
    "train_lengths_comments = list(map(lambda comment: len(comment.split()), train_comments))\n",
    "test_lengths_comments = list(map(lambda comment: len(comment.split()), test_comments))\n",
    "\n",
    "\n",
    "print(\"Mean length of train comments in WORDS:\", np.array(train_lengths_comments).mean())\n",
    "print(\"Mean length of test comments in WORDS:\", np.array(test_lengths_comments).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 624
    },
    "id": "ZE1M3fsOCkrv",
    "outputId": "d80b6946-b7d3-4346-c286-2b882728e9ec"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15, 8))\n",
    "plt.xlabel(\"Length of the function\", fontsize=20)\n",
    "plt.ylabel(\"Density\", fontsize=20)\n",
    "ax = sns.distplot(train_lengths_comments, color='#007b7f', label='Functions from train set', kde_kws=dict(linewidth=3))\n",
    "ax.patch.set_edgecolor('black')  \n",
    "ax.patch.set_linewidth('2')\n",
    "ax2 = sns.distplot(test_lengths_comments, color='g', label='Functions from test set', kde_kws=dict(linewidth=3))\n",
    "ax2.patch.set_edgecolor('black')  \n",
    "ax2.patch.set_linewidth('2')\n",
    "plt.setp(ax.patches, linewidth=3)\n",
    "plt.title(\"Comments lengths distributions\", fontsize=30)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HSr5Q0n3DLZu"
   },
   "source": [
    "# Words distribution for comments:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hr5rZ52DDY2o",
    "outputId": "a4ce8409-8c03-438d-a0c8-82f358e77be5"
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "nltk.download('stopwords')\n",
    "stop_words = set(stopwords.words('english') + ['ha', 'wa'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "a-GFIiM8DRve"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')\n",
    "%matplotlib inline\n",
    "\n",
    "count_vectorizer = CountVectorizer(stop_words=stop_words, min_df=10)\n",
    "comments_data = count_vectorizer.fit_transform(comments)\n",
    "\n",
    "functions_vectorizer = CountVectorizer(stop_words=stop_words, min_df=10)\n",
    "functions_data = functions_vectorizer.fit_transform(functions)\n",
    "\n",
    "def plot_most_common_words(count_data, count_vectorizer, n_words):\n",
    "    words = count_vectorizer.get_feature_names()\n",
    "    total_counts = np.zeros(len(words))\n",
    "    for t in count_data:\n",
    "        total_counts+=t.toarray()[0]\n",
    "    \n",
    "    count_dict = (zip(words, total_counts))\n",
    "    count_dict = sorted(count_dict, key=lambda x:x[1], reverse=True)[0:n_words]\n",
    "    words = [w[0] for w in count_dict]\n",
    "    counts = [w[1] for w in count_dict]\n",
    "    x_pos = np.arange(len(words)) \n",
    "    \n",
    "    plt.figure(2, figsize=(15, 15/1.6180))\n",
    "    plt.subplot(title='Most common words')\n",
    "    sns.set_context(\"notebook\", font_scale=1.25, rc={\"lines.linewidth\": 2.5})\n",
    "    sns.barplot(x_pos, counts, palette='husl')\n",
    "    plt.xticks(x_pos, words, rotation=90, fontsize=15) \n",
    "    plt.xlabel('words')\n",
    "    plt.ylabel('counts')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 706
    },
    "id": "Mp5nze3XDWLN",
    "outputId": "6748d140-a977-472a-eabc-d10c1e135652"
   },
   "outputs": [],
   "source": [
    "plot_most_common_words(comments_data, count_vectorizer, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 725
    },
    "id": "64x9ktjwDtWF",
    "outputId": "eecc862f-c570-4337-df11-2fe0c8a4efb9"
   },
   "outputs": [],
   "source": [
    "plot_most_common_words(functions_data, functions_vectorizer, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MyDy8SOu7KZf"
   },
   "source": [
    "# Consider WordCloud for functions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 417
    },
    "id": "RtoUJkPQ7KZi",
    "outputId": "c587753b-6516-42c5-e48e-e58090a2695e"
   },
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud\n",
    "\n",
    "all_texts = ','.join(functions)\n",
    "wordcloud = WordCloud(background_color=\"white\", max_words=5000, contour_width=3, \n",
    "                      contour_color='steelblue', width=800, height=400)\n",
    "\n",
    "wordcloud.generate(all_texts)\n",
    "wordcloud.to_image()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4vAM3012-726"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "EDA-Python150k.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "scs-ext",
   "language": "python",
   "name": "scs-ext"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}