Apache_Tika_Final.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Apache_Tika_Final.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Be4VKSWXq7gT",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        },
        "outputId": "541da406-ad31-47c0-8359-8fbaad68f6ff"
      },
      "source": [
        "#Setup\n",
        "!pip install tika\n",
        "!pip install pandas\n",
        "import pandas as pd\n",
        "import tika\n",
        "tika.initVM()\n",
        "from tika import parser\n",
        "import glob\n",
        "from google.colab import drive"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tika in /usr/local/lib/python3.6/dist-packages (1.24)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from tika) (49.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from tika) (2.23.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->tika) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->tika) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->tika) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->tika) (1.24.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (1.0.5)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from pandas) (1.18.5)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.6.1->pandas) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c5hWIqYo0vjU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        },
        "outputId": "c1f22a04-ce0c-4fad-d730-9484ba389249"
      },
      "source": [
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "emH8fek_FToI"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bT7c8zVorVBs",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "outputId": "881b7cf5-c014-4514-8a4a-6f98fd856e20"
      },
      "source": [
        "my_pdf = glob.glob(\"/content/drive/My Drive/d4g_database/all_pdf/*.pdf\")\n",
        "print(my_pdf)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JHMyaAKE1KYW"
      },
      "source": [
        "def pdf_meta():\n",
        "  pdfs = glob.glob(\"/content/drive/My Drive/d4g_database/all_pdf/*.pdf\")\n",
        "  meta_list = []\n",
        "  content_list = []\n",
        "\n",
        "  for item in pdfs:\n",
        "    parsed = parser.from_file(item)\n",
        "    meta_list.append(parsed[\"metadata\"])\n",
        "    content_list.append(parsed['content'])\n",
        "  return meta_list, content_list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u0-w9VnkxYis"
      },
      "source": [
        "metadata, contentdata = pdf_meta()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FuBcQjUQ48mt"
      },
      "source": [
        "def df_pdfs(meta, keywords):\n",
        "  unique_id = []\n",
        "  title_list = []\n",
        "  date_list = []\n",
        "  content_list = keywords\n",
        "  \n",
        "  [title_list.append(d['title']) if 'title' in d else title_list.append('null') for d in meta]\n",
        "  [date_list.append(d['date']) if 'date' in d else date_list.append('null') for d in meta]\n",
        "  counter = 0\n",
        "  for i in meta:\n",
        "    unique_id.append(counter)\n",
        "    counter += 1\n",
        "  \n",
        "  df = pd.DataFrame()\n",
        "  df['ID'] = unique_id\n",
        "  df['Title'] = title_list\n",
        "  df['Date'] = date_list\n",
        "  df['Content'] = content_list\n",
        "\n",
        "\n",
        "\n",
        "  return df\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0EFMcgDJ5B_j"
      },
      "source": [
        "pdf_df = df_pdfs(metadata, contentdata)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "If5WJTQUtJrF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 47
        },
        "outputId": "f0fb46b1-4c83-4439-c9c0-d15a0c7d6a24"
      },
      "source": [
        "pdf_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>Title</th>\n",
              "      <th>Date</th>\n",
              "      <th>Content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [ID, Title, Date, Content]\n",
              "Index: []"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-XCeSZCiFrRO"
      },
      "source": [
        "import sqlite3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hlq7N2-aFz8H"
      },
      "source": [
        "conn = sqlite3.connect('TestDB1.db')\n",
        "c = conn.cursor()\n",
        "\n",
        "c.execute('CREATE TABLE PDFS (ID text, Title text, Date text, Content text)')\n",
        "conn.commit()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9W8cRekevb7n"
      },
      "source": [
        "pdf_df.to_sql('PDFS', conn, if_exists='replace', index = False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}