representación_de_documentos.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "representación_de_documentos.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/merrecalde/rio_2020/blob/master/representaci%C3%B3n_de_documentos.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D9y1c9WSTe1q",
        "colab_type": "text"
      },
      "source": [
        "# Notebook: representación_de_documentos.ipynb\n",
        "\n",
        "### Se muestra cómo vectorizar (representar en formato vector) 3 documentos (strings) sencillos, utilizando como términos (features) a 1) las palabras, 2) bi-gramas de palabras y 3) tri-gramas de caracteres. Para las palabras, se muestran distintos tipos de pesados de los términos"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tPdiKu5r52TS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "documentos = [\"pintaron el banco de la plaza\",\n",
        "              \"te paso el programa, ejecútalo paso por paso\",\n",
        "              \"sentado en el banco, miraba si el banco abría\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qHAgk6VX7PrW",
        "colab_type": "code",
        "outputId": "78712fbe-3507-46b2-a352-6c67b097cb42",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "print(type(documentos))\n",
        "type(documentos[0])\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'list'>\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "str"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JqINbnVYUuc3",
        "colab_type": "text"
      },
      "source": [
        "## 1) Comenzamos con palabras como features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4yx4Eymw7WBJ",
        "colab_type": "code",
        "outputId": "fb55ca5b-32c3-4aff-e12a-b5e0aae69c0d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "vect = CountVectorizer()\n",
        "vect.fit(documentos)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
              "                dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
              "                lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
              "                ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
              "                strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                tokenizer=None, vocabulary=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xH6WOq1S8cVu",
        "colab_type": "code",
        "outputId": "e78de516-6ce7-46ca-ec12-738f8c6335ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        }
      },
      "source": [
        "print(\"Tamaño de Vocabulario: {}\".format(len(vect.vocabulary_)))\n",
        "print(\"Contenido del Vocabulario: {}\".format(vect.vocabulary_))\n",
        "print(\"Features: {}\".format(vect.get_feature_names()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tamaño de Vocabulario: 16\n",
            "Contenido del Vocabulario: {'pintaron': 9, 'el': 4, 'banco': 1, 'de': 2, 'la': 6, 'plaza': 10, 'te': 15, 'paso': 8, 'programa': 12, 'ejecútalo': 3, 'por': 11, 'sentado': 13, 'en': 5, 'miraba': 7, 'si': 14, 'abría': 0}\n",
            "Features: ['abría', 'banco', 'de', 'ejecútalo', 'el', 'en', 'la', 'miraba', 'paso', 'pintaron', 'plaza', 'por', 'programa', 'sentado', 'si', 'te']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GLg4_q85IUGA",
        "colab_type": "text"
      },
      "source": [
        "### Ahora transforma los documentos en vectores"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WBiSyMJ895bm",
        "colab_type": "code",
        "outputId": "34c42d7c-e807-48db-f244-b519dabf314b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 381
        }
      },
      "source": [
        "bolsa_de_palabras = vect.transform(documentos)\n",
        "print(\"Bolsa de palabras: {}\\n\".format(bolsa_de_palabras))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Bolsa de palabras:   (0, 1)\t1\n",
            "  (0, 2)\t1\n",
            "  (0, 4)\t1\n",
            "  (0, 6)\t1\n",
            "  (0, 9)\t1\n",
            "  (0, 10)\t1\n",
            "  (1, 3)\t1\n",
            "  (1, 4)\t1\n",
            "  (1, 8)\t3\n",
            "  (1, 11)\t1\n",
            "  (1, 12)\t1\n",
            "  (1, 15)\t1\n",
            "  (2, 0)\t1\n",
            "  (2, 1)\t2\n",
            "  (2, 4)\t2\n",
            "  (2, 5)\t1\n",
            "  (2, 7)\t1\n",
            "  (2, 13)\t1\n",
            "  (2, 14)\t1\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Z0jwZ_sBLqJ",
        "colab_type": "code",
        "outputId": "a362f486-3cca-447b-ed23-4174f59c038b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "print(\"Bolsa de palabras: {}\".format(repr(bolsa_de_palabras)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Bolsa de palabras: <3x16 sparse matrix of type '<class 'numpy.int64'>'\n",
            "\twith 19 stored elements in Compressed Sparse Row format>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-tg5hy7BBXcW",
        "colab_type": "code",
        "outputId": "81e16f29-d863-4f2d-b266-0f872b13199b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "source": [
        "print(\"Matriz documentos - términos:\\n{}\".format(bolsa_de_palabras.toarray()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Matriz documentos - términos:\n",
            "[[0 1 1 0 1 0 1 0 0 1 1 0 0 0 0 0]\n",
            " [0 0 0 1 1 0 0 0 3 0 0 1 1 0 0 1]\n",
            " [1 2 0 0 2 1 0 1 0 0 0 0 0 1 1 0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "miMl_CnkHb5m",
        "colab_type": "text"
      },
      "source": [
        "### 1.1) Pesado binario"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W8-OipDJHtVr",
        "colab_type": "code",
        "outputId": "45cd6eec-4162-42cf-f698-5e2e706cc758",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "source": [
        "vect_bin = CountVectorizer(binary=True)\n",
        "vect_bin.fit(documentos)\n",
        "bow_bin= vect_bin.transform(documentos)\n",
        "print(\"Matriz documentos - términos (pesado binario):\\n{}\".format(bow_bin.toarray()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Matriz documentos - términos (pesado binario):\n",
            "[[0 1 1 0 1 0 1 0 0 1 1 0 0 0 0 0]\n",
            " [0 0 0 1 1 0 0 0 1 0 0 1 1 0 0 1]\n",
            " [1 1 0 0 1 1 0 1 0 0 0 0 0 1 1 0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BKcb2E2AKLFL",
        "colab_type": "text"
      },
      "source": [
        "### 1.2) Pesado tf-idf"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N0p_knYPMja1",
        "colab_type": "code",
        "outputId": "139fbe09-e030-4037-e86b-4a3d19be73a5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145
        }
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "vect_tf_idf = TfidfVectorizer()\n",
        "vect_tf_idf.fit(documentos)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
              "                dtype=<class 'numpy.float64'>, encoding='utf-8',\n",
              "                input='content', lowercase=True, max_df=1.0, max_features=None,\n",
              "                min_df=1, ngram_range=(1, 1), norm='l2', preprocessor=None,\n",
              "                smooth_idf=True, stop_words=None, strip_accents=None,\n",
              "                sublinear_tf=False, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                tokenizer=None, use_idf=True, vocabulary=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qiO6BrdBBmtH",
        "colab_type": "text"
      },
      "source": [
        "Con los parámetros por defecto usa la frecuencia del término clásico (directa) $tf(t,d)$, y la $idf$ es \"smootheada\" quedando $idf(t) = \\log(\\frac{1 + n}{1 + fd(t)}) + 1$. También aplica la normalización l2, es decir, realiza la normalización coseno. En síntesis, el pesado final es $$ w_{d,t} = tf(t,d) \\times (\\log (\\frac{1 + n}{1 + fd(t)}) + 1)$$\n",
        "\n",
        "y luego este pesado es normalizado $$ \\frac{w_{d,t}}{\\sqrt{\\sum_{t^\\prime \\in T} w_{d,t^\\prime}}} $$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lHTsruKKNc30",
        "colab_type": "code",
        "outputId": "bebbc67a-7331-4462-acf1-2af71a2c9c4e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        }
      },
      "source": [
        "bow_tf_idf_1 = vect_tf_idf.transform(documentos)\n",
        "print(\"Matriz documentos - términos (pesado tf-idf por defecto):\\n{}\".format(bow_tf_idf_1.toarray()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Matriz documentos - términos (pesado tf-idf por defecto):\n",
            "[[0.         0.34261996 0.45050407 0.         0.26607496 0.\n",
            "  0.45050407 0.         0.         0.45050407 0.45050407 0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.2737023  0.16165299 0.\n",
            "  0.         0.         0.82110689 0.         0.         0.2737023\n",
            "  0.2737023  0.         0.         0.2737023 ]\n",
            " [0.33885833 0.51542099 0.         0.         0.40027038 0.33885833\n",
            "  0.         0.33885833 0.         0.         0.         0.\n",
            "  0.         0.33885833 0.33885833 0.        ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i82OfdQ2HiIy",
        "colab_type": "text"
      },
      "source": [
        "Si le sacamos el \"smootheado\" al $idf$ (parámetro *smooth_idf=False*) queda $idf(t) = \\log(\\frac{n}{fd(t)}) + 1$ y el pesado final es $$ w_{d,t} = tf(t,d) \\times (\\log (\\frac{n}{fd(t)}) + 1)$$\n",
        "\n",
        "y luego este pesado es normalizado $$ \\frac{w_{d,t}}{\\sqrt{\\sum_{t^\\prime \\in T} w_{d,t^\\prime}}} $$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zdn50bfuIl0W",
        "colab_type": "code",
        "outputId": "2045905e-ab0a-41c6-e004-0d7b73f46d24",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        }
      },
      "source": [
        "vect_tf_idf_2 = TfidfVectorizer(smooth_idf=False)\n",
        "vect_tf_idf_2.fit(documentos)\n",
        "bow_tf_idf_2 = vect_tf_idf_2.transform(documentos)\n",
        "print(\"Matriz documentos - términos (pesado tf-idf (sin suavizado de idf)):\\n{}\".format(bow_tf_idf_2.toarray()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Matriz documentos - términos (pesado tf-idf (sin suavizado de idf)):\n",
            "[[0.         0.30972091 0.46246905 0.         0.22036898 0.\n",
            "  0.46246905 0.         0.         0.46246905 0.46246905 0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.27495928 0.13101957 0.\n",
            "  0.         0.         0.82487784 0.         0.         0.27495928\n",
            "  0.27495928 0.         0.         0.27495928]\n",
            " [0.36032154 0.48262307 0.         0.         0.34339029 0.36032154\n",
            "  0.         0.36032154 0.         0.         0.         0.\n",
            "  0.         0.36032154 0.36032154 0.        ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kTsPLH0DOFYj",
        "colab_type": "text"
      },
      "source": [
        "Si en lugar de trabajar directamente con la frecuencia del término, aplicamos un escalado \"sublineal\" (reemplazando tf por 1 + log(tf)),  usamos el parámetro *sublinear_tf=True*) y el pesado final (con los restantes valores por defecto) es $$ w_{d,t} = (1 + \\log(tf(t,d))) \\times (\\log (\\frac{1 + n}{1 + fd(t)}) + 1)$$\n",
        "\n",
        "y luego este pesado es normalizado $$ \\frac{w_{d,t}}{\\sqrt{\\sum_{t^\\prime \\in T} w_{d,t^\\prime}}} $$\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1YZtVRXgNrsi",
        "colab_type": "code",
        "outputId": "ce904d64-9920-4f3b-f1a4-59d032e45bf1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        }
      },
      "source": [
        "vect_tf_idf_3 = TfidfVectorizer(sublinear_tf=True)\n",
        "vect_tf_idf_3.fit(documentos)\n",
        "bow_tf_idf_3 = vect_tf_idf_3.transform(documentos)\n",
        "print(\"Matriz documentos - términos (pesado tf-idf (con frecuencia de termino sub-lineal)):\\n{}\".format(bow_tf_idf_3.toarray()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Matriz documentos - términos (pesado tf-idf (con frecuencia de termino sub-lineal)):\n",
            "[[0.         0.34261996 0.45050407 0.         0.26607496 0.\n",
            "  0.45050407 0.         0.         0.45050407 0.45050407 0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.         0.         0.         0.33800375 0.19963046 0.\n",
            "  0.         0.         0.70933881 0.         0.         0.33800375\n",
            "  0.33800375 0.         0.         0.33800375]\n",
            " [0.36135891 0.46531539 0.         0.         0.36135891 0.36135891\n",
            "  0.         0.36135891 0.         0.         0.         0.\n",
            "  0.         0.36135891 0.36135891 0.        ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x0akRSbaRpzu",
        "colab_type": "text"
      },
      "source": [
        "Como se puede observar, la mayoria de las codificaciones SMART pueden ser logradas con este vectorizador, a excepción de aquellas 2 que en la frecuencia del término utilizan las máximas frecuencias de un término en el documento (m** y a**)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OKAezTEjVMqP",
        "colab_type": "text"
      },
      "source": [
        "## 2) Ahora usamos bi-gramas de palabras como features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "15d94107-fa1f-4c63-948a-77d00cbd45c0",
        "id": "HCEn9w6aR_3A",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "source": [
        "vect2 = CountVectorizer(ngram_range=(2, 2))\n",
        "vect2.fit(documentos)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
              "                dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
              "                lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
              "                ngram_range=(2, 2), preprocessor=None, stop_words=None,\n",
              "                strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                tokenizer=None, vocabulary=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "01c55eb6-38c9-4de0-fdd3-cba4d7948d68",
        "id": "i_zvMWsoR_3W",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        }
      },
      "source": [
        "print(\"Tamaño de Vocabulario: {}\".format(len(vect2.vocabulary_)))\n",
        "print(\"Contenido del Vocabulario: {}\".format(vect2.vocabulary_))\n",
        "print(\"Features: {}\".format(vect2.get_feature_names()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tamaño de Vocabulario: 18\n",
            "Contenido del Vocabulario: {'pintaron el': 12, 'el banco': 5, 'banco de': 1, 'de la': 3, 'la plaza': 8, 'te paso': 17, 'paso el': 10, 'el programa': 6, 'programa ejecútalo': 14, 'ejecútalo paso': 4, 'paso por': 11, 'por paso': 13, 'sentado en': 15, 'en el': 7, 'banco miraba': 2, 'miraba si': 9, 'si el': 16, 'banco abría': 0}\n",
            "Features: ['banco abría', 'banco de', 'banco miraba', 'de la', 'ejecútalo paso', 'el banco', 'el programa', 'en el', 'la plaza', 'miraba si', 'paso el', 'paso por', 'pintaron el', 'por paso', 'programa ejecútalo', 'sentado en', 'si el', 'te paso']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "b50b4439-7ac0-4d40-c14c-a48cb707c17b",
        "id": "CHltZBGiR_3b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 381
        }
      },
      "source": [
        "bolsa_de_bi_pal = vect2.transform(documentos)\n",
        "print(\"Bolsa de bi-gramas de palabras: \\n{}\".format(bolsa_de_bi_pal))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Bolsa de bi-gramas de palabras: \n",
            "  (0, 1)\t1\n",
            "  (0, 3)\t1\n",
            "  (0, 5)\t1\n",
            "  (0, 8)\t1\n",
            "  (0, 12)\t1\n",
            "  (1, 4)\t1\n",
            "  (1, 6)\t1\n",
            "  (1, 10)\t1\n",
            "  (1, 11)\t1\n",
            "  (1, 13)\t1\n",
            "  (1, 14)\t1\n",
            "  (1, 17)\t1\n",
            "  (2, 0)\t1\n",
            "  (2, 2)\t1\n",
            "  (2, 5)\t2\n",
            "  (2, 7)\t1\n",
            "  (2, 9)\t1\n",
            "  (2, 15)\t1\n",
            "  (2, 16)\t1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "d0fa7bc3-f32c-4e73-fa8c-8ec95261677f",
        "id": "9RzAroQkR_3e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "source": [
        "print(\"Matriz documentos - términos (bi-gramas de palabras):\\n{}\".format(bolsa_de_bi_pal.toarray()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Matriz documentos - términos (bi-gramas de palabras):\n",
            "[[0 1 0 1 0 1 0 0 1 0 0 0 1 0 0 0 0 0]\n",
            " [0 0 0 0 1 0 1 0 0 0 1 1 0 1 1 0 0 1]\n",
            " [1 0 1 0 0 2 0 1 0 1 0 0 0 0 0 1 1 0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WfPjb10EVgF9",
        "colab_type": "text"
      },
      "source": [
        "## 3) Ahora las features son tri-gramas de caracteres"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2unilOH1CRuV",
        "colab_type": "code",
        "outputId": "65fe0184-b501-4039-dfd5-49f9c84309fe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "source": [
        "vect3 = CountVectorizer(analyzer='char',ngram_range=(3, 3))\n",
        "vect3.fit(documentos)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(analyzer='char', binary=False, decode_error='strict',\n",
              "                dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
              "                lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
              "                ngram_range=(3, 3), preprocessor=None, stop_words=None,\n",
              "                strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                tokenizer=None, vocabulary=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jge6xfrTQLCy",
        "colab_type": "code",
        "outputId": "a4f34951-c61b-4323-c351-0a294e264787",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        }
      },
      "source": [
        "print(\"Tamaño de Vocabulario: {}\".format(len(vect3.vocabulary_)))\n",
        "print(\"Contenido del Vocabulario: {}\".format(vect3.vocabulary_))\n",
        "print(\"Features: {}\".format(vect3.get_feature_names()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tamaño de Vocabulario: 84\n",
            "Contenido del Vocabulario: {'pin': 66, 'int': 44, 'nta': 56, 'tar': 81, 'aro': 24, 'ron': 74, 'on ': 63, 'n e': 54, ' el': 4, 'el ': 39, 'l b': 47, ' ba': 1, 'ban': 28, 'anc': 23, 'nco': 55, 'co ': 30, 'o d': 58, ' de': 2, 'de ': 33, 'e l': 35, ' la': 6, 'la ': 49, 'a p': 15, ' pl': 9, 'pla': 67, 'laz': 50, 'aza': 26, 'te ': 82, 'e p': 36, ' pa': 8, 'pas': 65, 'aso': 25, 'so ': 78, 'o e': 59, 'l p': 48, ' pr': 11, 'pro': 69, 'rog': 73, 'ogr': 62, 'gra': 42, 'ram': 72, 'ama': 22, 'ma,': 52, 'a, ': 17, ', e': 13, ' ej': 3, 'eje': 38, 'jec': 46, 'ecú': 37, 'cút': 32, 'úta': 83, 'tal': 80, 'alo': 21, 'lo ': 51, 'o p': 60, ' po': 10, 'por': 68, 'or ': 64, 'r p': 70, 'sen': 76, 'ent': 41, 'tad': 79, 'ado': 20, 'do ': 34, ' en': 5, 'en ': 40, 'co,': 31, 'o, ': 61, ', m': 14, ' mi': 7, 'mir': 53, 'ira': 45, 'rab': 71, 'aba': 18, 'ba ': 27, 'a s': 16, ' si': 12, 'si ': 77, 'i e': 43, 'o a': 57, ' ab': 0, 'abr': 19, 'brí': 29, 'ría': 75}\n",
            "Features: [' ab', ' ba', ' de', ' ej', ' el', ' en', ' la', ' mi', ' pa', ' pl', ' po', ' pr', ' si', ', e', ', m', 'a p', 'a s', 'a, ', 'aba', 'abr', 'ado', 'alo', 'ama', 'anc', 'aro', 'aso', 'aza', 'ba ', 'ban', 'brí', 'co ', 'co,', 'cút', 'de ', 'do ', 'e l', 'e p', 'ecú', 'eje', 'el ', 'en ', 'ent', 'gra', 'i e', 'int', 'ira', 'jec', 'l b', 'l p', 'la ', 'laz', 'lo ', 'ma,', 'mir', 'n e', 'nco', 'nta', 'o a', 'o d', 'o e', 'o p', 'o, ', 'ogr', 'on ', 'or ', 'pas', 'pin', 'pla', 'por', 'pro', 'r p', 'rab', 'ram', 'rog', 'ron', 'ría', 'sen', 'si ', 'so ', 'tad', 'tal', 'tar', 'te ', 'úta']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "842K08WsQavx",
        "colab_type": "code",
        "outputId": "d3701510-61b6-4278-b07a-5e1ce9ae77de",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 963
        }
      },
      "source": [
        "bolsa_de_tri_car = vect3.transform(documentos)\n",
        "print(\"Bolsa de tri-gramas de caracteres: \\n{}\".format(bolsa_de_tri_car))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Bolsa de tri-gramas de caracteres: \n",
            "  (0, 1)\t1\n",
            "  (0, 2)\t1\n",
            "  (0, 4)\t1\n",
            "  (0, 6)\t1\n",
            "  (0, 9)\t1\n",
            "  (0, 15)\t1\n",
            "  (0, 23)\t1\n",
            "  (0, 24)\t1\n",
            "  (0, 26)\t1\n",
            "  (0, 28)\t1\n",
            "  (0, 30)\t1\n",
            "  (0, 33)\t1\n",
            "  (0, 35)\t1\n",
            "  (0, 39)\t1\n",
            "  (0, 44)\t1\n",
            "  (0, 47)\t1\n",
            "  (0, 49)\t1\n",
            "  (0, 50)\t1\n",
            "  (0, 54)\t1\n",
            "  (0, 55)\t1\n",
            "  (0, 56)\t1\n",
            "  (0, 58)\t1\n",
            "  (0, 63)\t1\n",
            "  (0, 66)\t1\n",
            "  (0, 67)\t1\n",
            "  :\t:\n",
            "  (2, 23)\t2\n",
            "  (2, 27)\t1\n",
            "  (2, 28)\t2\n",
            "  (2, 29)\t1\n",
            "  (2, 30)\t1\n",
            "  (2, 31)\t1\n",
            "  (2, 34)\t1\n",
            "  (2, 39)\t2\n",
            "  (2, 40)\t1\n",
            "  (2, 41)\t1\n",
            "  (2, 43)\t1\n",
            "  (2, 45)\t1\n",
            "  (2, 47)\t2\n",
            "  (2, 53)\t1\n",
            "  (2, 54)\t1\n",
            "  (2, 55)\t2\n",
            "  (2, 56)\t1\n",
            "  (2, 57)\t1\n",
            "  (2, 59)\t1\n",
            "  (2, 61)\t1\n",
            "  (2, 71)\t1\n",
            "  (2, 75)\t1\n",
            "  (2, 76)\t1\n",
            "  (2, 77)\t1\n",
            "  (2, 79)\t1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IbYvKVDKRCdX",
        "colab_type": "code",
        "outputId": "4cd60e5d-9ca8-42a9-bfe6-dc7e5f086075",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        }
      },
      "source": [
        "print(\"Matriz documentos - términos (tri-gramas de caracteres):\\n{}\".format(bolsa_de_tri_car.toarray()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Matriz documentos - términos (tri-gramas de caracteres):\n",
            "[[0 1 1 0 1 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 1 1 0 1 0 1 0 1 0 0 1 0 1\n",
            "  0 0 0 1 0 0 0 0 1 0 0 1 0 1 1 0 0 0 1 1 1 0 1 0 0 0 0 1 0 0 1 1 0 0 0 0\n",
            "  0 0 1 0 0 0 0 0 0 1 0 0]\n",
            " [0 0 0 1 1 0 0 0 3 0 1 1 0 1 0 0 0 1 0 0 0 1 1 0 0 3 0 0 0 0 0 0 1 0 0 0\n",
            "  1 1 1 1 0 0 1 0 0 0 1 0 1 0 0 1 1 0 0 0 0 0 0 1 2 0 1 0 1 3 0 0 1 1 1 0\n",
            "  1 1 0 0 0 0 2 0 1 0 1 1]\n",
            " [1 2 0 0 2 1 0 1 0 0 0 0 1 0 1 0 1 0 1 1 1 0 0 2 0 0 0 1 2 1 1 1 0 0 1 0\n",
            "  0 0 0 2 1 1 0 1 0 1 0 2 0 0 0 0 0 1 1 2 1 1 0 1 0 1 0 0 0 0 0 0 0 0 0 1\n",
            "  0 0 0 1 1 1 0 1 0 0 0 0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wwnzdKpYXlfW",
        "colab_type": "text"
      },
      "source": [
        "## 4) Tri-gramas de caracteres (con límites de palabra)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "256649ac-febd-4129-8bbf-418bf109c820",
        "id": "Vk2Gz7O9YcqK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "source": [
        "vect4 = CountVectorizer(analyzer='char_wb',ngram_range=(3, 3))\n",
        "vect4.fit(documentos)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(analyzer='char_wb', binary=False, decode_error='strict',\n",
              "                dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
              "                lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
              "                ngram_range=(3, 3), preprocessor=None, stop_words=None,\n",
              "                strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                tokenizer=None, vocabulary=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "dd4866e0-eed5-4c27-a1c2-6b6d5a128351",
        "id": "KyD8lX2kYcqU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        }
      },
      "source": [
        "print(\"Tamaño de Vocabulario: {}\".format(len(vect4.vocabulary_)))\n",
        "print(\"Contenido del Vocabulario: {}\".format(vect4.vocabulary_))\n",
        "print(\"Features: {}\".format(vect4.get_feature_names()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tamaño de Vocabulario: 74\n",
            "Contenido del Vocabulario: {' pi': 9, 'pin': 55, 'int': 40, 'nta': 49, 'tar': 69, 'aro': 23, 'ron': 62, 'on ': 52, ' el': 4, 'el ': 36, ' ba': 1, 'ban': 27, 'anc': 22, 'nco': 48, 'co ': 29, ' de': 2, 'de ': 32, ' la': 6, 'la ': 43, ' pl': 10, 'pla': 56, 'laz': 44, 'aza': 25, 'za ': 71, ' te': 15, 'te ': 70, ' pa': 8, 'pas': 54, 'aso': 24, 'so ': 66, ' pr': 12, 'pro': 58, 'rog': 61, 'ogr': 51, 'gra': 39, 'ram': 60, 'ama': 21, 'ma,': 46, 'a, ': 16, ' ej': 3, 'eje': 35, 'jec': 42, 'ecú': 34, 'cút': 31, 'úta': 73, 'tal': 68, 'alo': 20, 'lo ': 45, ' po': 11, 'por': 57, 'or ': 53, ' se': 13, 'sen': 64, 'ent': 38, 'tad': 67, 'ado': 19, 'do ': 33, ' en': 5, 'en ': 37, 'co,': 30, 'o, ': 50, ' mi': 7, 'mir': 47, 'ira': 41, 'rab': 59, 'aba': 17, 'ba ': 26, ' si': 14, 'si ': 65, ' ab': 0, 'abr': 18, 'brí': 28, 'ría': 63, 'ía ': 72}\n",
            "Features: [' ab', ' ba', ' de', ' ej', ' el', ' en', ' la', ' mi', ' pa', ' pi', ' pl', ' po', ' pr', ' se', ' si', ' te', 'a, ', 'aba', 'abr', 'ado', 'alo', 'ama', 'anc', 'aro', 'aso', 'aza', 'ba ', 'ban', 'brí', 'co ', 'co,', 'cút', 'de ', 'do ', 'ecú', 'eje', 'el ', 'en ', 'ent', 'gra', 'int', 'ira', 'jec', 'la ', 'laz', 'lo ', 'ma,', 'mir', 'nco', 'nta', 'o, ', 'ogr', 'on ', 'or ', 'pas', 'pin', 'pla', 'por', 'pro', 'rab', 'ram', 'rog', 'ron', 'ría', 'sen', 'si ', 'so ', 'tad', 'tal', 'tar', 'te ', 'za ', 'ía ', 'úta']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "a8d27abd-6435-4b30-b38c-b38f0f5bbfbc",
        "id": "bAah-ECQYcqc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 963
        }
      },
      "source": [
        "bolsa_de_tri_car_lim = vect4.transform(documentos)\n",
        "print(\"Bolsa de tri-gramas de caracteres con limite palabra: \\n{}\".format(bolsa_de_tri_car_lim))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Bolsa de tri-gramas de caracteres con limite palabra: \n",
            "  (0, 1)\t1\n",
            "  (0, 2)\t1\n",
            "  (0, 4)\t1\n",
            "  (0, 6)\t1\n",
            "  (0, 9)\t1\n",
            "  (0, 10)\t1\n",
            "  (0, 22)\t1\n",
            "  (0, 23)\t1\n",
            "  (0, 25)\t1\n",
            "  (0, 27)\t1\n",
            "  (0, 29)\t1\n",
            "  (0, 32)\t1\n",
            "  (0, 36)\t1\n",
            "  (0, 40)\t1\n",
            "  (0, 43)\t1\n",
            "  (0, 44)\t1\n",
            "  (0, 48)\t1\n",
            "  (0, 49)\t1\n",
            "  (0, 52)\t1\n",
            "  (0, 55)\t1\n",
            "  (0, 56)\t1\n",
            "  (0, 62)\t1\n",
            "  (0, 69)\t1\n",
            "  (0, 71)\t1\n",
            "  (1, 3)\t1\n",
            "  :\t:\n",
            "  (2, 14)\t1\n",
            "  (2, 17)\t1\n",
            "  (2, 18)\t1\n",
            "  (2, 19)\t1\n",
            "  (2, 22)\t2\n",
            "  (2, 26)\t1\n",
            "  (2, 27)\t2\n",
            "  (2, 28)\t1\n",
            "  (2, 29)\t1\n",
            "  (2, 30)\t1\n",
            "  (2, 33)\t1\n",
            "  (2, 36)\t2\n",
            "  (2, 37)\t1\n",
            "  (2, 38)\t1\n",
            "  (2, 41)\t1\n",
            "  (2, 47)\t1\n",
            "  (2, 48)\t2\n",
            "  (2, 49)\t1\n",
            "  (2, 50)\t1\n",
            "  (2, 59)\t1\n",
            "  (2, 63)\t1\n",
            "  (2, 64)\t1\n",
            "  (2, 65)\t1\n",
            "  (2, 67)\t1\n",
            "  (2, 72)\t1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "8dc03ec4-6e02-4c62-ff27-a03351367509",
        "id": "tmBAree6Ycqh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        }
      },
      "source": [
        "print(\"Matriz documentos - términos (tri-gramas de caracteres - con limite palabra):\\n{}\".format(bolsa_de_tri_car_lim.toarray()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Matriz documentos - términos (tri-gramas de caracteres - con limite palabra):\n",
            "[[0 1 1 0 1 0 1 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 1 1 0 1 0 1 0 1 0 0 1 0 0 0\n",
            "  1 0 0 0 1 0 0 1 1 0 0 0 1 1 0 0 1 0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 1 0 1\n",
            "  0 0]\n",
            " [0 0 0 1 1 0 0 0 3 0 0 1 1 0 0 1 1 0 0 0 1 1 0 0 3 0 0 0 0 0 0 1 0 0 1 1\n",
            "  1 0 0 1 0 0 1 0 0 1 1 0 0 0 0 1 0 1 3 0 0 1 1 0 1 1 0 0 0 0 3 0 1 0 1 0\n",
            "  0 1]\n",
            " [1 2 0 0 2 1 0 1 0 0 0 0 0 1 1 0 0 1 1 1 0 0 2 0 0 0 1 2 1 1 1 0 0 1 0 0\n",
            "  2 1 1 0 0 1 0 0 0 0 0 1 2 1 1 0 0 0 0 0 0 0 0 1 0 0 0 1 1 1 0 1 0 0 0 0\n",
            "  1 0]]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}