15_stc_to_text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## I. Decoding STC using Python's library pdfplumber"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example use of library pdfplumber on one STC pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "United States of America\n",
      "Department of Transportation\n",
      "Federal Aviation Administration\n",
      "Supplemental Type Certificate\n",
      "Number:\n",
      "SA2322CE-D\n",
      "This certificate issued to: Honeywell International Inc.\n",
      "21111 N 19th Ave Phoenix\n",
      "Arizona 85027\n",
      "certifies that the change in the type design for the following product with the limitations and conditions therefore as\n",
      "4b\n",
      "specified hereon meets the airworthiness requirements of Part of the CivilAirRegulations.\n",
      "OriginalProduct–TypeCertificateNumber: Make:AvionisMarcelDassault(AMD)\n",
      "Model:\n",
      "A7EU FanjetFalconSeriesC,D,E,F\n",
      "Mystere-Falcon20-C5,20-D5,20-E5,20-F5\n",
      "DescriptionofTypeDesignChange:\n",
      "InstallationofdualBendix/KingKFC400AutomaticFlightControlSystems.\n",
      "REQUIREDDATA:1.MasterDrawingList155-9513-01,Rev.7,dated5-90and2.AirplaneFlightManualSupplement\n",
      "006-00499-0000,Rev.1,dated6-14-90orlaterFAAapprovedrevisionto1or2.\n",
      "LimitationsandConditions:\n",
      "1.CompliancemustbeshownwithapplicableServiceBulletinsandairplanemodificationsaslistedontheRequired\n",
      "Aircraft Modifications Document 155-09552-0000, Rev. 1, dated 10-89 or later revision.\n",
      "(Continued on Continuation Sheet) If the holder agrees to permit another person to use this certificate to alter the\n",
      "product, the holder shall give the other person written evidence of that permission\n",
      "This certificate and the supporting data which is the basis for approval shall remain in effect until surrendered,\n",
      "suspended, and revoked or a termination date is otherwise established by the Administrator of the Federal Aviation\n",
      "Administration.\n",
      "DateofApplication:11-9-89 DateReissued: 12-20-94;2-28-00;3-31-20\n",
      "DateofIssuance: 4-19-89 DateAmended:\n",
      "6-14-90\n",
      "By Direction of the Administrator\n",
      "Signature\n",
      "Title RichardB.Sleigh\n",
      "ODAAdministrator,ODA-602216-NM\n",
      "Anyalterationof this certificateispunishablebyafineofnotexceeding$1,000,orimprisonmentnotexceeding3years,orboth. Thiscertificate may be\n",
      "transferred ormadeavailableto thirdpersons bylicensingagreementsinaccordancewith14CFR21.47.Possessionof thisSupplementalType\n",
      "Certificate(STC)documentbypersonsotherthantheSTCholder doesnotconstituterightstothedesigndatanortoalteranaircraft,aircraftengine,or\n",
      "propeller.TheSTC’ssupportingdocumentation (drawings, instructions, specifications,flightmanualsupplements,etc.)is thepropertyoftheSTC\n",
      "holder.AnSTCholderwhoallowsapersontousetheSTCtoalteranaircraft,aircraftengine,or propellermustprovidethatpersonwithwritten\n",
      "permissionacceptabletotheFAA.(Ref.14CFR21.120).\n",
      "FAAForm8110-2(5/14) Page1of3\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pdfplumber\n",
    "import os\n",
    "\n",
    "with pdfplumber.open(os.path.join(os.getcwd(),'data','STC','raw data','pdf','SA2322CE-D__Current__A174E8B29C3F2AA6862585D800611438.pdf')) as pdf: # SB04185CH  SR04557NY\n",
    "    pdf_pages = [str(this_page.extract_text())+'\\n' for this_page in pdf.pages]\n",
    "\n",
    "print(pdf_pages[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## II. Create STC text database using Google Vision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(os.getcwd(), 'google_project_name.txt'), 'r') as f:\n",
    "    google_project_name = f.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Upload local files to google storage\n",
    "https://console.cloud.google.com/storage/browser/faa-drs;tab=objects?project=drs-stc&prefix=&forceOnObjectsSortingFiltering=false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from google.cloud import storage\n",
    "storage_client = storage.Client()\n",
    "blobs = storage_client.list_blobs(google_project_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stc/\n",
      "stc/SA2322CE-D__Current__A174E8B29C3F2AA6862585D800611438.pdf\n"
     ]
    }
   ],
   "source": [
    "for blob in blobs:\n",
    "   print(blob.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "\n",
    "\n",
    "def list_blobs(bucket_name):\n",
    "    \"\"\"Lists all the blobs in the bucket.\"\"\"\n",
    "    # bucket_name = \"your-bucket-name\"\n",
    "\n",
    "    storage_client = storage.Client()\n",
    "\n",
    "    # Note: Client.list_blobs requires at least package version 1.17.0.\n",
    "    blobs = storage_client.list_blobs(bucket_name)\n",
    "\n",
    "    # Note: The call returns a response only when the iterator is consumed.\n",
    "    #for blob in blobs:\n",
    "    #    print(blob.name)\n",
    "\n",
    "    return [blob.name for blob in blobs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "\n",
    "\n",
    "def upload_blob(bucket_name, source_file_name, destination_blob_name):\n",
    "    \"\"\"Uploads a file to the bucket.\"\"\"\n",
    "    # The ID of your GCS bucket\n",
    "    # bucket_name = \"your-bucket-name\"\n",
    "    # The path to your file to upload\n",
    "    # source_file_name = \"local/path/to/file\"\n",
    "    # The ID of your GCS object\n",
    "    # destination_blob_name = \"storage-object-name\"\n",
    "\n",
    "    storage_client = storage.Client()\n",
    "    bucket = storage_client.bucket(bucket_name)\n",
    "    blob = bucket.blob(destination_blob_name)\n",
    "\n",
    "    # Optional: set a generation-match precondition to avoid potential race conditions\n",
    "    # and data corruptions. The request to upload is aborted if the object's\n",
    "    # generation number does not match your precondition. For a destination\n",
    "    # object that does not yet exist, set the if_generation_match precondition to 0.\n",
    "    # If the destination object already exists in your bucket, set instead a\n",
    "    # generation-match precondition using its generation number.\n",
    "    generation_match_precondition = 0\n",
    "\n",
    "    blob.upload_from_filename(source_file_name, if_generation_match=generation_match_precondition)\n",
    "\n",
    "    print(\n",
    "        f\"File {source_file_name} uploaded to {destination_blob_name}.\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['SA2322CE-D__Current__A174E8B29C3F2AA6862585D800611438.pdf']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_blobs(google_project_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "\n",
    "dl_dir = os.path.join(os.path.join(os.getcwd(),'database','data','stc','pdf'))\n",
    "\n",
    "uploaded_pdfs = list_blobs(google_project_name)\n",
    "\n",
    "uploaded_pdfs = [os.path.basename(file) for file in list_blobs(google_project_name) if file[:4]==\"pdf/\"]\n",
    "list_of_real_pdfs = [file for file in glob.glob(os.path.join(dl_dir,\"*.pdf\")) if os.path.getsize(file)>200 and os.path.basename(file) not in uploaded_pdfs]\n",
    "\n",
    "for filename in list_of_real_pdfs:\n",
    "    upload_blob(google_project_name, \n",
    "                filename, \n",
    "                \"pdf/\"+os.path.basename(filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. PDF to Text using Google Vision\n",
    "From https://cloud.google.com/vision/docs/pdf#vision_text_detection_pdf_gcs-python:\n",
    "* https://cloud.google.com/storage/pricing\n",
    "* https://cloud.google.com/vision/docs/pdf\n",
    "* gcloud projects create drs-stc\n",
    "* gcloud config set project drs-stc\n",
    "* gcloud services enable vision.googleapis.com\n",
    "* gcloud projects add-iam-policy-binding drs-stc --member=\"user:victor.girondin@gmail.com\" --role=roles/owner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def async_detect_document(gcs_source_uri, gcs_destination_uri):\n",
    "    \"\"\"OCR with PDF/TIFF as source files on GCS\"\"\"\n",
    "    import json\n",
    "    import re\n",
    "    from google.cloud import vision\n",
    "    from google.cloud import storage\n",
    "\n",
    "    # Supported mime_types are: 'application/pdf' and 'image/tiff'\n",
    "    mime_type = \"application/pdf\"\n",
    "\n",
    "    # How many pages should be grouped into each json output file.\n",
    "    batch_size = 2\n",
    "\n",
    "    client = vision.ImageAnnotatorClient()\n",
    "\n",
    "    feature = vision.Feature(type_=vision.Feature.Type.DOCUMENT_TEXT_DETECTION) # DOCUMENT_TEXT_DETECTION\n",
    "\n",
    "    gcs_source = vision.GcsSource(uri=gcs_source_uri)\n",
    "    input_config = vision.InputConfig(gcs_source=gcs_source, mime_type=mime_type)\n",
    "\n",
    "    gcs_destination = vision.GcsDestination(uri=gcs_destination_uri)\n",
    "    output_config = vision.OutputConfig(\n",
    "        gcs_destination=gcs_destination, batch_size=batch_size\n",
    "    )\n",
    "\n",
    "    async_request = vision.AsyncAnnotateFileRequest(\n",
    "        features=[feature], input_config=input_config, output_config=output_config\n",
    "    )\n",
    "\n",
    "    operation = client.async_batch_annotate_files(requests=[async_request])\n",
    "\n",
    "    print(\"Waiting for the operation to finish.\")\n",
    "    operation.result(timeout=420)\n",
    "\n",
    "    # Once the request has completed and the output has been\n",
    "    # written to GCS, we can list all the output files.\n",
    "    storage_client = storage.Client()\n",
    "\n",
    "    match = re.match(r\"gs://([^/]+)/(.+)\", gcs_destination_uri)\n",
    "    bucket_name = match.group(1)\n",
    "    prefix = match.group(2)\n",
    "\n",
    "    bucket = storage_client.get_bucket(bucket_name)\n",
    "\n",
    "    # List objects with the given prefix, filtering out folders.\n",
    "    blob_list = [\n",
    "        blob\n",
    "        for blob in list(bucket.list_blobs(prefix=prefix))\n",
    "        if not blob.name.endswith(\"/\")\n",
    "    ]\n",
    "    print(\"Output files:\")\n",
    "    for blob in blob_list:\n",
    "        print(blob.name)\n",
    "\n",
    "    # Process the first output file from GCS.\n",
    "    # Since we specified batch_size=2, the first response contains\n",
    "    # the first two pages of the input file.\n",
    "    output = blob_list[0]\n",
    "\n",
    "    json_string = output.download_as_bytes().decode(\"utf-8\")\n",
    "    response = json.loads(json_string)\n",
    "\n",
    "    # The actual response for the first page of the input file.\n",
    "    first_page_response = response[\"responses\"][0]\n",
    "    annotation = first_page_response[\"fullTextAnnotation\"]\n",
    "\n",
    "    # Here we print the full text from the first page.\n",
    "    # The response contains more information:\n",
    "    # annotation/pages/blocks/paragraphs/words/symbols\n",
    "    # including confidence scores and bounding boxes\n",
    "    # print(\"Full text:\\n\")\n",
    "    # print(annotation[\"text\"])\n",
    "\n",
    "    outputs = []\n",
    "    for blob in blob_list:\n",
    "        response = json.loads(blob.download_as_string())\n",
    "        outputs.append(\"\\n\\n\".join([page_response['fullTextAnnotation']['text'] \n",
    "                                    for page_response in response['responses'] \n",
    "                                    if ('fullTextAnnotation' in page_response)]))\n",
    "\n",
    "    return \"\\n\\n\".join(outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A simple example just try"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for the operation to finish.\n",
      "Output files:\n",
      "bbd1_output-1-to-2.json\n",
      "bbd1_output-3-to-3.json\n",
      "FEDER\n",
      "NOLL\n",
      "This certificate issued to: Honeywell International Inc.\n",
      "21111 N 19th Ave Phoenix\n",
      "Arizona 85027\n",
      "A7EU\n",
      "United States of America\n",
      "Department of Transportation\n",
      "Federal Aviation Administration\n",
      "Supplemental Type Certificate\n",
      "certifies that the change in the type design for the following product with the limitations and conditions therefore as\n",
      "specified hereon meets the airworthiness requirements of Part 46 of the Civil Air Regulations.\n",
      "Original Product Type Certificate Number:\n",
      "Description of Type Design Change:\n",
      "Number: SA2322CE-D\n",
      "Date of Application: 11-9-89\n",
      "Make: Avionis Marcel Dassault (AMD)\n",
      "Model:\n",
      "Installation of dual Bendix/King KFC 400 Automatic Flight Control Systems.\n",
      "REQUIRED DATA: 1. Master Drawing List 155-9513-01, Rev. 7, dated 5-90 and 2. Airplane Flight Manual Supplement\n",
      "006-00499-0000, Rev. 1, dated 6-14-90 or later FAA approved revision to 1 or 2.\n",
      "Date of Issuance: 4-19-89\n",
      "Limitations and Conditions:\n",
      "1. Compliance must be shown with applicable Service Bulletins and airplane modifications as listed on the Required\n",
      "Aircraft Modifications Document 155-09552-0000, Rev. 1, dated 10-89 or later revision.\n",
      "(Continued on Continuation Sheet) If the holder agrees to permit another person to use this certificate to alter the\n",
      "product, the holder shall give the other person written evidence of that permission\n",
      "Fanjet Falcon Series C, D, E, F\n",
      "Mystere-Falcon 20-C5, 20-D5, 20-E5, 20-F5\n",
      "This certificate and the supporting data which is the basis for approval shall remain in effect until surrendered,\n",
      "suspended, and revoked or a termination date is otherwise established by the Administrator of the Federal Aviation\n",
      "Administration.\n",
      "Date Reissued: 12-20-94; 2-28-00, 3-31-20\n",
      "Date Amended:\n",
      "Signature\n",
      "6-14-90\n",
      "By Direction of the Administrator\n",
      "Tile Richard B. Sleigh\n",
      "ODA Administrator, ODA-602216-NM\n",
      "Any alteration of this certificate is punishable by a fine of not exceeding $1,000, or imprisonment not exceeding 3 years, or both. This certificate may be\n",
      "transferred or made available to third persons by licensing agreements in accordance with 14 CFR 21.47. Possession of this Supplemental Type\n",
      "Certificate (STC) document by persons other than the STC holder does not constitute rights to the design data nor to alter an aircraft, aircraft engine, or\n",
      "propeller. The STC's supporting documentation (drawings, instructions, specifications, fight manual supplements, etc.) is the property of the STC\n",
      "holder. An STC holder who allows a person to use the STC to after an aircraft, aircraft engine, or propeller must provide that person with written\n",
      "permission acceptable to the FAA. (Ref. 14 CFR 21.1201\n",
      "FAA Fom 8110-2 (5/14)\n",
      "Page 1 of 3\n",
      "\n",
      "FEDER\n",
      "VIATION\n",
      "TRATION\n",
      "United States of America\n",
      "Department of Transportation\n",
      "Federal Aviation Administration\n",
      "Supplemental Type Certificate\n",
      "INSTRUCTIONS: The transfer endorsement below may be used to notify the appropriate FAA Aircraft Centification Office of the transfer of this\n",
      "Supplemental Type Certificate. The FAA will reissue the certificate in the name of the transferee and forward it to him.\n",
      "Transfer Endorsement\n",
      "Transfer the ownership of Supplemental Type Certificate Number:\n",
      "To (Name and address of transferee)\n",
      "From (Name and address of grantor)\n",
      "Extent of Authority (if licensing agreement):\n",
      "Date of transfer:\n",
      "Signature of grantor\n",
      "Any alteration of this certificate is punishable by a fine of not exceeding $1,000, or imprisonment not exceeding 3 years, or both. This certificate may be\n",
      "transferred or made available to third persons by licensing agreements in accordance with 14 CFR 21.47. Possession of this Supplemental Type\n",
      "Certificate (STC) document by persons other than the STC holder does not constitute rights to the design data nor to alter an aircraft, aircraft engine, or\n",
      "propeller. The STC's supporting documentation (drawings, instructions, specifications, fight manual supplements, etc.) is the property of the STC\n",
      "holder. An STC holder who allows a person to use the STC to after an aircraft, aircraft engine, or propeller must provide that person with written\n",
      "permission acceptable to the FAA. (Ref. 14 CFR 21.120).\n",
      "FAA Fom 8110-2 (5/14)\n",
      "Page 2 of 3\n",
      "\n",
      "FEDER\n",
      "NOUV\n",
      "United States of America\n",
      "Department of Transportation\n",
      "Federal Aviation Administration\n",
      "Supplemental Type Certificate\n",
      "(Continuation Sheet)\n",
      "Number: SA2322CE-D\n",
      "Limitations and Conditions: (continued)\n",
      "2. This Approval included an evaluation and finding of acceptance of the interface between the Bendix/King EFS 10 EFIS, KFC\n",
      "400 AFCS, KNS 660 FMS, KAD 480 Air Data and Series III Communication/Navigation/Identification (CNI) Systems.\n",
      "3. This modification is approved only with the concurrent installation of modifications as defined by STC SA2317CE-D\n",
      "(Bendix/King KAD 480 Air Data) and STC SA2315CE-D, Configuration \"B\" (Bendix/King EFS 10 EFIS with Bendix/King KAH 460\n",
      "AHRS).\n",
      "4. This approval should not be extended to other specific airplanes of this model on which other previously approved\n",
      "modifications are incorporated, unless it is determined that the interrelationship between this change and any of those\n",
      "previously approved modifications will introduce no adverse effect upon the airworthiness of that airplane.\n",
      "END\n",
      "Any alteration of this certificate is punishable by a fine of not exceeding $1,000, or imprisonment not exceeding 3 years, or both. This certificate may be\n",
      "transferred or made available to third persons by licensing agreements in accordance with 14 CFR 21.47. Possession of this Supplemental Type\n",
      "Certificate (STC) document by persons other than the STC holder does not constitute rights to the design data nor to after an aircraft, aircraft engine, ori\n",
      "propeller. The STC's supporting documentation (drawings, instructions, specifications, fight manual supplements, etc.) is the property of the STC\n",
      "holder. An STC holder who allows a person to use the STC to alter an aircraft, aircraft engine, or propeller must provide that person with written\n",
      "permission acceptable to the FAA. (Rel. 14 CFR 21.120).\n",
      "FAA Form 8110-2 (5/14)\n",
      "Page 3 of\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "import re\n",
    "from google.cloud import vision\n",
    "from google.cloud import storage\n",
    "\n",
    "toto = async_detect_document(\"gs://faa-drs/bbd1.pdf\", \"gs://faa-drs/bbd1_\")\n",
    "\n",
    "print(toto)\n",
    "\n",
    "text_dir = r'C:\\Users\\victor\\Downloads'#'C:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\database\\data\\stc\\text-from-pdf'\n",
    "\n",
    "# with open(os.path.join(text_dir, \"SA2322CE-D__Current__A174E8B29C3F2AA6862585D800611438.txt\"), 'w') as f:\n",
    "#     f.write(toto)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Go throught the pdfs in google cloud, apply vision api and retrieve the text (download_as_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import glob\n",
    "import random\n",
    "\n",
    "text_dir = os.path.join(os.getcwd(),'database','data','stc','text-from-pdf')\n",
    "\n",
    "list_of_texts = [os.path.basename(file)[:-4] for file in glob.glob(os.path.join(text_dir,\"*.txt\"))]\n",
    "all_blobs = list_blobs(google_project_name)\n",
    "\n",
    "print(str(len([blob for blob in all_blobs if ((blob[:4]==\"pdf/\") and (blob[-4:]==\".pdf\") and (os.path.basename(blob)[:-4] not in list_of_texts))]))+\" missing over \"+str(len(all_blobs)))\n",
    "\n",
    "random.shuffle(all_blobs)\n",
    "\n",
    "for blob in all_blobs:\n",
    "    if (blob[:4]==\"pdf/\") and (blob[-4:]==\".pdf\") and (os.path.basename(blob)[:-4] not in list_of_texts):\n",
    "        try:\n",
    "            print(blob)\n",
    "            extracted_text = async_detect_document(\"gs://faa-drs/\"+blob,\n",
    "                                                re.sub(\"faa-drs/pdf\",\n",
    "                                                        \"faa-drs/text-from-pdf\",\n",
    "                                                        \"gs://faa-drs/\"+blob[:-4])\n",
    "                                                )\n",
    "            with open(os.path.join(text_dir, os.path.basename(blob[:-4])+\".txt\"), 'w') as f:\n",
    "                f.write(extracted_text)\n",
    "        except:\n",
    "            print(\"it failed :(\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve the json from Google cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "from google.cloud import storage\n",
    "\n",
    "json_dir = os.path.join(os.getcwd(),'database','data','stc','json-from-pdf')\n",
    "list_of_jsons = [os.path.basename(file)[:-5] for file in glob.glob(os.path.join(json_dir,\"*.json\"))]\n",
    "\n",
    "storage_client = storage.Client()\n",
    "bucket = storage_client.bucket(google_project_name)\n",
    "\n",
    "for blob_name in all_blobs:\n",
    "    if (blob_name[:14]==\"text-from-pdf/\") and (blob_name[-5:]==\".json\") and (os.path.basename(blob_name)[:-5] not in list_of_jsons):\n",
    "        try:\n",
    "            print(blob_name)\n",
    "            blob = bucket.blob(blob_name)\n",
    "            blob.download_to_filename(os.path.join(json_dir, os.path.basename(blob_name)))\n",
    "        except:\n",
    "            print(\"it failed :(\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Go throught the locally stored pdfs and retrieve the text (alternative using download_as_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(77661, 26)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "df_stc = pd.read_excel(os.path.join(os.getcwd(),'database','stc.xlsx'))\n",
    "df_stc = df_stc.drop_duplicates()\n",
    "df_stc['drs:stcHolder'] = df_stc['drs:stcHolder'].map(lambda x: x.replace(\", Inc.\",\"\").replace(\", Inc\",\"\").replace(\" Inc.\",\"\").replace(\" Inc\",\"\"))\n",
    "\n",
    "print(df_stc.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "\n",
    "storage_client = storage.Client()\n",
    "bucket = storage_client.bucket(\"faa-drs\")\n",
    "\n",
    "blob_list = [blob for blob in list(bucket.list_blobs(prefix='text-from-pdf')) if not blob.name.endswith('/')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import json\n",
    "\n",
    "text_dir = os.path.join(os.getcwd(),'database','data','stc','text-from-pdf-alternate')\n",
    "processed_pdfs = glob.glob(os.path.join(text_dir,'*.txt'))\n",
    "\n",
    "for documentGuid in list(df_stc['documentGuid']):\n",
    "    matching_blobs = [blob for blob in blob_list if documentGuid in blob.name]\n",
    "\n",
    "    if len([val for val in processed_pdfs if documentGuid in val])==0:\n",
    "        outputs = []\n",
    "\n",
    "        for blob in matching_blobs:\n",
    "            response = json.loads(blob.download_as_text())\n",
    "            outputs.append(\"\\n\\n\".join([page_response['fullTextAnnotation']['text'] for page_response in response['responses'] if ('fullTextAnnotation' in page_response)]))\n",
    "            \n",
    "        outputs = \"\\n\\n\".join(outputs)\n",
    "\n",
    "        with open(os.path.join(text_dir, blob.name[14:-5].split(\"output\")[0]+\".txt\"), 'w') as f:\n",
    "            f.write(outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## III. Google Vision OCR on all DRS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Upload local files to google storage\n",
    "https://console.cloud.google.com/storage/browser/faa-drs;tab=objects?project=drs-stc&prefix=&forceOnObjectsSortingFiltering=false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "from google.cloud import storage\n",
    "\n",
    "\n",
    "def list_blobs(bucket_name):\n",
    "    \"\"\"Lists all the blobs in the bucket.\"\"\"\n",
    "    # bucket_name = \"your-bucket-name\"\n",
    "\n",
    "    storage_client = storage.Client()\n",
    "\n",
    "    # Note: Client.list_blobs requires at least package version 1.17.0.\n",
    "    blobs = storage_client.list_blobs(bucket_name)\n",
    "\n",
    "    # Note: The call returns a response only when the iterator is consumed.\n",
    "    #for blob in blobs:\n",
    "    #    print(blob.name)\n",
    "\n",
    "    return [blob.name for blob in blobs]                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "\n",
    "\n",
    "def upload_blob(bucket_name, source_file_name, destination_blob_name):\n",
    "    \"\"\"Uploads a file to the bucket.\"\"\"\n",
    "    # The ID of your GCS bucket\n",
    "    # bucket_name = \"your-bucket-name\"\n",
    "    # The path to your file to upload\n",
    "    # source_file_name = \"local/path/to/file\"\n",
    "    # The ID of your GCS object\n",
    "    # destination_blob_name = \"storage-object-name\"\n",
    "\n",
    "    storage_client = storage.Client()\n",
    "    bucket = storage_client.bucket(bucket_name)\n",
    "    blob = bucket.blob(destination_blob_name)\n",
    "\n",
    "    # Optional: set a generation-match precondition to avoid potential race conditions\n",
    "    # and data corruptions. The request to upload is aborted if the object's\n",
    "    # generation number does not match your precondition. For a destination\n",
    "    # object that does not yet exist, set the if_generation_match precondition to 0.\n",
    "    # If the destination object already exists in your bucket, set instead a\n",
    "    # generation-match precondition using its generation number.\n",
    "    generation_match_precondition = 0\n",
    "\n",
    "    blob.upload_from_filename(source_file_name, if_generation_match=generation_match_precondition)\n",
    "\n",
    "    print(\n",
    "        f\"File {source_file_name} uploaded to {destination_blob_name}.\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "\n",
    "dl_dir = os.path.join(os.path.join(os.getcwd(),'database','data','stc','pdf'))\n",
    "\n",
    "uploaded_pdfs = list_blobs(\"faa-drs\")\n",
    "\n",
    "uploaded_pdfs = [os.path.basename(file) for file in list_blobs(\"faa-drs\") if file[:4]==\"pdf/\"]\n",
    "list_of_real_pdfs = [file for file in glob.glob(os.path.join(dl_dir,\"*.pdf\")) if os.path.getsize(file)>200 and os.path.basename(file) not in uploaded_pdfs]\n",
    "\n",
    "for filename in list_of_real_pdfs:\n",
    "    upload_blob(\"faa-drs\", \n",
    "                filename, \n",
    "                \"pdf/\"+os.path.basename(filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. PDF to Text using Google Vision\n",
    "From https://cloud.google.com/vision/docs/pdf#vision_text_detection_pdf_gcs-python:\n",
    "* https://cloud.google.com/storage/pricing\n",
    "* https://cloud.google.com/vision/pricing\n",
    "* https://cloud.google.com/vision/docs/pdf\n",
    "* gcloud projects create drs-stc\n",
    "* gcloud config set project drs-stc\n",
    "* gcloud services enable vision.googleapis.com\n",
    "* gcloud projects add-iam-policy-binding drs-stc --member=\"user:victor.girondin@gmail.com\" --role=roles/owner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def async_detect_document(gcs_source_uri, gcs_destination_uri):\n",
    "    \"\"\"OCR with PDF/TIFF as source files on GCS\"\"\"\n",
    "    import json\n",
    "    import re\n",
    "    from google.cloud import vision\n",
    "    from google.cloud import storage\n",
    "\n",
    "    # Supported mime_types are: 'application/pdf' and 'image/tiff'\n",
    "    mime_type = 'application/pdf'\n",
    "\n",
    "    # How many pages should be grouped into each json output file.\n",
    "    batch_size = 2\n",
    "\n",
    "    client = vision.ImageAnnotatorClient()\n",
    "\n",
    "    feature = vision.Feature(\n",
    "        type_=vision.Feature.Type.DOCUMENT_TEXT_DETECTION)\n",
    "\n",
    "    gcs_source = vision.GcsSource(uri=gcs_source_uri)\n",
    "    input_config = vision.InputConfig(\n",
    "        gcs_source=gcs_source, mime_type=mime_type)\n",
    "\n",
    "    gcs_destination = vision.GcsDestination(uri=gcs_destination_uri)\n",
    "    output_config = vision.OutputConfig(\n",
    "        gcs_destination=gcs_destination, batch_size=batch_size)\n",
    "\n",
    "    async_request = vision.AsyncAnnotateFileRequest(\n",
    "        features=[feature], input_config=input_config,\n",
    "        output_config=output_config)\n",
    "\n",
    "    operation = client.async_batch_annotate_files(\n",
    "        requests=[async_request])\n",
    "\n",
    "    #print('Waiting for the operation to finish.')\n",
    "    operation.result(timeout=420)\n",
    "\n",
    "    # Once the request has completed and the output has been\n",
    "    # written to GCS, we can list all the output files.\n",
    "    storage_client = storage.Client()\n",
    "\n",
    "    match = re.match(r'gs://([^/]+)/(.+)', gcs_destination_uri)\n",
    "    bucket_name = match.group(1)\n",
    "    prefix = match.group(2)\n",
    "\n",
    "    bucket = storage_client.get_bucket(bucket_name)\n",
    "\n",
    "    # List objects with the given prefix, filtering out folders.\n",
    "    blob_list = [blob for blob in list(bucket.list_blobs(\n",
    "        prefix=prefix)) if not blob.name.endswith('/')]\n",
    "    print('Output files:')\n",
    "    for blob in blob_list:\n",
    "        print(blob.name)\n",
    "\n",
    "    # Process the first output file from GCS.\n",
    "    # Since we specified batch_size=2, the first response contains\n",
    "    # the first two pages of the input file.\n",
    "    #output = blob_list[0]\n",
    "\n",
    "    #json_string = output.download_as_string()\n",
    "    #response = json.loads(json_string)\n",
    "\n",
    "    # The actual response for the first page of the input file.\n",
    "    #first_page_response = response['responses'][0]\n",
    "    #annotation = first_page_response['fullTextAnnotation']\n",
    "\n",
    "    # Here we print the full text from the first page.\n",
    "    # The response contains more information:\n",
    "    # annotation/pages/blocks/paragraphs/words/symbols\n",
    "    # including confidence scores and bounding boxes\n",
    "    #print('Full text:\\n')\n",
    "    #print(annotation['text'])\n",
    "\n",
    "    outputs = []\n",
    "    for blob in blob_list:\n",
    "        response = json.loads(blob.download_as_string())\n",
    "        outputs.append(\"\\n\\n\".join([page_response['fullTextAnnotation']['text'] \n",
    "                                    for page_response in response['responses'] \n",
    "                                    if ('fullTextAnnotation' in page_response)]))\n",
    "\n",
    "    return \"\\n\\n\".join(outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Go throught the pdfs in google cloud, apply vision api and retrieve the text (download_as_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "df_stc = pd.read_excel(os.path.join(os.getcwd(),'database','stc.xlsx'))\n",
    "df_stc = df_stc.drop_duplicates()\n",
    "df_stc['drs:stcHolder'] = df_stc['drs:stcHolder'].map(lambda x: x.replace(\", Inc.\",\"\").replace(\", Inc\",\"\").replace(\" Inc.\",\"\").replace(\" Inc\",\"\"))\n",
    "\n",
    "print(df_stc.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "\n",
    "storage_client = storage.Client()\n",
    "bucket = storage_client.bucket(\"faa-drs\")\n",
    "\n",
    "blob_list = [blob for blob in list(bucket.list_blobs(prefix='text-from-pdf')) if not blob.name.endswith('/')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import json\n",
    "\n",
    "text_dir = os.path.join(os.getcwd(),'database','data','stc','text-from-pdf-alternate')\n",
    "processed_pdfs = glob.glob(os.path.join(text_dir,'*.txt'))\n",
    "\n",
    "for documentGuid in list(df_stc['documentGuid']):\n",
    "    matching_blobs = [blob for blob in blob_list if documentGuid in blob.name]\n",
    "\n",
    "    if len([val for val in processed_pdfs if documentGuid in val])==0:\n",
    "        outputs = []\n",
    "\n",
    "        for blob in matching_blobs:\n",
    "            response = json.loads(blob.download_as_text())\n",
    "            outputs.append(\"\\n\\n\".join([page_response['fullTextAnnotation']['text'] for page_response in response['responses'] if ('fullTextAnnotation' in page_response)]))\n",
    "            \n",
    "        outputs = \"\\n\\n\".join(outputs)\n",
    "\n",
    "        with open(os.path.join(text_dir, blob.name[14:-5].split(\"output\")[0]+\".txt\"), 'w') as f:\n",
    "            f.write(outputs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "8c49a9e6d6b122d0bc46f085b0f9b84866f77188fdff3252b8a9eae176833897"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}