capture_mac.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from PyQt5 import QtWidgets, QtCore, QtGui\n",
    "from PyQt5.QtWidgets import (QSystemTrayIcon, QApplication, QLabel, QMainWindow, QPushButton, QWidget, \n",
    "                             QVBoxLayout, QTextEdit, QDialog, QComboBox, QAction, QLineEdit, QLabel, QInputDialog)\n",
    "from PyQt5.QtGui import QIcon, QFont\n",
    "from PyQt5.QtCore import Qt, QCoreApplication, QSettings, QPoint\n",
    "import tkinter as tk\n",
    "import PIL.Image\n",
    "import numpy as np\n",
    "import cv2\n",
    "import io\n",
    "import json\n",
    "import inspect\n",
    "import ctypes\n",
    "from io import BytesIO\n",
    "import glob\n",
    "import os\n",
    "from matplotlib import pyplot as plt\n",
    "from IPython.display import clear_output, Image, display\n",
    "from google.cloud.vision_v1 import types\n",
    "from google.cloud import vision\n",
    "import mss\n",
    "import pyperclip\n",
    "import mss.tools\n",
    "import qt_utils\n",
    "from PyQt5.QtWidgets import QMessageBox\n",
    "\n",
    "# os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"] = \"C:/Users/poorv/Downloads/ocr_scripts/keys/direct-outlook-270501-c05c2d97d1c6.json\"\n",
    "os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"] = \"/Users/poorv/Downloads/ocr/service_credentials.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'தோட்டம்\\nஇது ஓர் அழகான தோட்டம். இந்தத் தோட்டத்தில் நிறைய\\nபூச்செடிகளும் பல பழ மரங்களும் இருக்கின்றன. அதனால் நிறைய\\nபறவைகளையும்\\nவண்டுகளையும்\\nஇங்கே\\nபார்க்கலாம்.\\nசின்னக்\\nகுழந்தைகள் தங்கள் பெற்றோருடன் இங்கே வருவார்கள். இந்தத்\\nதோட்டத்தில் மான்கள் கூட இருக்கின்றன. இந்தத் தோட்டத்தைப் பராமரிக்க\\nநல்ல தோட்டக்காரர்கள் இருக்கிறார்கள். காலையிலிருந்து இரவுவரை\\nஅவர்கள் இங்கே இருப்பார்கள். இங்கே வரும் குழந்தைகளுக்கு\\nவாசனையான பூக்களையும் ருசியான பழங்களையும் கொடுப்பர்கள்.\\nஅதனால் குழந்தைகள் இங்கே வர ஆசைப்படுவார்கள்.\\nBharathy, UCB\\n139\\n'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def send_text_clipboard(text):\n",
    "    pyperclip.copy(text)\n",
    "    spam = pyperclip.paste()\n",
    "    \n",
    "def dtlh(path, lh):\n",
    "    \"\"\"Detects text in the file.\"\"\"\n",
    "    client = vision.ImageAnnotatorClient()\n",
    "    if type(path) == str:\n",
    "        with io.open(path, 'rb') as image_file:\n",
    "            content = image_file.read()\n",
    "            image = vision.Image(content=content)\n",
    "    else:\n",
    "        image = types.Image(content=cv2.imencode('.jpg', path)[1].tobytes())\n",
    "        \n",
    "    response = client.text_detection(\n",
    "    image=image,\n",
    "    image_context={\"language_hints\": lh},\n",
    "    )\n",
    "    texts = response.text_annotations\n",
    "    if response.error.message:\n",
    "        raise Exception(\n",
    "            '{}\\nFor more info on error messages, check: '\n",
    "            'https://cloud.google.com/apis/design/errors'.format(\n",
    "                response.error.message))\n",
    "\n",
    "    try:\n",
    "        return texts[0].description\n",
    "    except IndexError:\n",
    "        print('no text in image')\n",
    "        \n",
    "dtlh(r\"/Users/poorv/Downloads/tamil/better pages/139.png\", [\"ta\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "width = 2560\n",
    "height = 1600\n",
    "\n",
    "class ocrWidget(QtWidgets.QWidget):\n",
    "#     clicked = QtCore.pyqtSignal(object)\n",
    "    \n",
    "    def __init__(self, parent = None, lang_hint = []):\n",
    "        super().__init__()\n",
    "        self.setGeometry(0, 0, width, height)\n",
    "        self.setWindowTitle(' ')\n",
    "        self.lang_hint = lang_hint\n",
    "        self.begin = QtCore.QPoint()\n",
    "        self.end = QtCore.QPoint()\n",
    "        self.parent = parent\n",
    "        self.setWindowOpacity(0.3)\n",
    "        QtWidgets.QApplication.setOverrideCursor(\n",
    "            QtGui.QCursor(QtCore.Qt.CrossCursor)\n",
    "        )\n",
    "        self.setWindowFlags(QtCore.Qt.FramelessWindowHint)\n",
    "        QtWidgets.QShortcut(\n",
    "            QtGui.QKeySequence(\"Escape\"), self, activated=self.on_Escape\n",
    "        )\n",
    "\n",
    "    def paintEvent(self, event):\n",
    "        qp = QtGui.QPainter(self)\n",
    "        qp.setPen(QtGui.QPen(QtGui.QColor('black'), 3))\n",
    "        qp.setBrush(QtGui.QColor(128, 128, 255, 128))\n",
    "        qp.drawRect(QtCore.QRect(self.begin, self.end))\n",
    "\n",
    "    def mousePressEvent(self, event):\n",
    "        self.begin = event.pos()\n",
    "        self.end = self.begin\n",
    "        self.update()\n",
    "    \n",
    "    @QtCore.pyqtSlot()\n",
    "    def on_Escape(self):\n",
    "        print(\"main esp exit\")\n",
    "#         self.close()\n",
    "        self.closeAndReturn()\n",
    "        \n",
    "\n",
    "    def mouseMoveEvent(self, event):\n",
    "        self.end = event.pos()\n",
    "        self.update()\n",
    "    \n",
    "    def closeAndReturn(self):\n",
    "        self.close()\n",
    "        self.parent.setWindowOpacity(1.)\n",
    "        return\n",
    "\n",
    "    def mouseReleaseEvent(self, event):\n",
    "        previous = self.parent\n",
    "        self.close()\n",
    "\n",
    "        x1 = min(self.begin.x(), self.end.x())\n",
    "        y1 = min(self.begin.y(), self.end.y())\n",
    "        x2 = max(self.begin.x(), self.end.x())\n",
    "        y2 = max(self.begin.y(), self.end.y())\n",
    "        \n",
    "        if (y1 == y2 and x1 == x2):\n",
    "            print(\"no region selected\")\n",
    "            self.closeAndReturn()\n",
    "        \n",
    "        with mss.mss() as sct:\n",
    "            print(\"lang_hint:\", self.lang_hint)\n",
    "            # The screen part to capture            \n",
    "            monitor = {\"top\": y1, \"left\":x1, \"width\": abs(x2 - x1) , \"height\": abs(y2 - y1)}\n",
    "            output = \"sct-{top}x{left}_{width}x{height}.png\".format(**monitor)\n",
    "            sct_img = sct.grab(monitor)\n",
    "            try:\n",
    "                img = PIL.Image.frombytes(\"RGB\", sct_img.size, sct_img.bgra, \"raw\", \"BGRX\")\n",
    "                img = cv2.cvtColor(np.array(img), cv2.COLOR_BGR2RGB)\n",
    "                im_gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)\n",
    "                ret,thresh = cv2.threshold(im_gray,127,255,cv2.THRESH_BINARY_INV)\n",
    "#                 ocr = dtlh(thresh, [\"hi\"])\n",
    "                ocr = dtlh(thresh, self.lang_hint)\n",
    "                popup = False\n",
    "                if ocr:\n",
    "                    send_text_clipboard(ocr)\n",
    "                    print(ocr)\n",
    "                    if popup:\n",
    "                        msg = QMessageBox()\n",
    "                        msg.setText(ocr)\n",
    "                        msg.setWindowTitle(\"Captured Text\")\n",
    "                        msg.setFont(QFont('Arial', 20)) \n",
    "                        x = msg.exec_()  # this will show our messagebox\n",
    "                self.parent.setWindowOpacity(1.)\n",
    "            except ValueError:\n",
    "                print(\"invalid region selection. try again\")\n",
    "                self.closeAndReturn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  OpenType support missing for \"Arial\", script 14\n",
      "  OpenType support missing for \"Arial\", script 11\n",
      "  OpenType support missing for \"Arial\", script 18\n",
      "  OpenType support missing for \"Arial\", script 32\n",
      "  OpenType support missing for \"Arial\", script 19\n",
      "  OpenType support missing for \"Arial\", script 13\n",
      "  OpenType support missing for \"Arial\", script 16\n",
      "  OpenType support missing for \"Arial\", script 17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "button clicked\n",
      "lang_hint: ['ta']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/4n/6vck65rd73d5bflh0f914j3m0000gn/T/ipykernel_73995/105359903.py:13: DeprecationWarning: tostring() is deprecated. Use tobytes() instead.\n",
      "  image = types.Image(content=cv2.imencode('.jpg', path)[1].tostring())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ஒரு காட்டில் விலங்குகள் எல்லாம் சேர்ந்து ஒரு யானையைத்\n",
      "தலைவனாகத் தேர்ந்தெடுத்தன. ஆனால் ஒரு மானுக்கு அது\n",
      "பிடிக்கவில்லை. அந்த மான் மற்ற விலங்குகளிடம், \"யானையுடைய\n",
      "உடம்பு ரொம்ப பெரிய உடம்பு; யானைக்குப் பெரிய தந்தம் இருக்கிறது.\n",
      ":\n",
      "யானை பார்க்க அசிங்கமாக இருக்கிறது. நான் பார்க்க அழகாக\n",
      "இருக்கிறேன். என்னைத் தலைவனாகத் தேர்ந்தெடுங்கள்” என்று\n",
      "சொன்னது. ஆனால் மற்ற மிருகங்களுக்கு அது பிடிக்கவில்லை.\n",
      "அதனால் மான் இந்த இடத்திலிருந்து வேறு இடத்துக்குப் போனது. ஒரு\n",
      "\n",
      "button clicked\n",
      "lang_hint: ['ta']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/4n/6vck65rd73d5bflh0f914j3m0000gn/T/ipykernel_73995/105359903.py:13: DeprecationWarning: tostring() is deprecated. Use tobytes() instead.\n",
      "  image = types.Image(content=cv2.imencode('.jpg', path)[1].tostring())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ஓநாய்\n",
      "வசமாக\n",
      "சிக்கிக்கொள்\n",
      "கழுதை\n",
      "6\n",
      "கடி\n",
      "விநாடி / வினாடி\n",
      "\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "0",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/poorv/.pyenv/versions/3.9.7/lib/python3.9/site-packages/IPython/core/interactiveshell.py:3452: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "lang_hints_dict = {u'Afrikaans - Afrikaans': \"af\", \n",
    "u'Albanian - shqip': \"sq\", \n",
    "u'Arabic - العربية': \"ar\", \n",
    "u'Armenian - Հայ': \"hy\", \n",
    "u'Belorussian - беларускі': \"be\", \n",
    "u'Bulgarian - български': \"bg\", \n",
    "u'Catalan - Català': \"ca\", \n",
    "u'Chinese - 普通话': \"zh\", \n",
    "u'Croatian - Hrvatski': \"hr\", \n",
    "u'Czech - Čeština': \"cs\", \n",
    "u'Danish - Dansk': \"da\", \n",
    "u'Dutch - Nederlands': \"nl\", \n",
    "u'English  - English': \"en\", \n",
    "u'Estonian - Eesti keel': \"et\", \n",
    "u'Filipino - Filipino': \"fil (or tl)\", \n",
    "u'Finnish - Suomi': \"fi\", \n",
    "u'French  - Français': \"fr\", \n",
    "u'German - Deutsch': \"de\", \n",
    "u'Greek - Ελληνικά': \"el\", \n",
    "u'Gujarati - ગુજરાતી': \"gu\", \n",
    "u'Hebrew - עברית': \"iw\", \n",
    "u'Hindi - हिन्दी': \"hi\", \n",
    "u'Hungarian - Magyar': \"hu\", \n",
    "u'Icelandic - Íslenska': \"is\", \n",
    "u'Indonesian - Bahasa Indonesia': \"id\", \n",
    "u'Italian - Italiano': \"it\", \n",
    "u'Japanese - 日本語': \"ja\", \n",
    "u'Kannada - ಕನ್ನಡ': \"kn\", \n",
    "u'Khmer - ភាសាខ្មែរ': \"km\", \n",
    "u'Korean - 한국어': \"ko\", \n",
    "u'Lao - ລາວ': \"lo\", \n",
    "u'Latvian - Latviešu': \"lv\", \n",
    "u'Lithuanian - Lietuvių': \"lt\", \n",
    "u'Macedonian - Македонски': \"mk\", \n",
    "u'Malay - Bahasa Melayu': \"ms\", \n",
    "u'Malayalam - മലയാളം': \"ml\", \n",
    "u'Marathi - मराठी': \"mr\", \n",
    "u'Nepali - नेपाली': \"ne\", \n",
    "u'Norwegian - Norsk': \"no\", \n",
    "u'Persian - فارسی': \"fa\", \n",
    "u'Polish - Polski': \"pl\", \n",
    "u'Portuguese - Português': \"pt\", \n",
    "u'Punjabi - ਪੰਜਾਬੀ': \"pa\", \n",
    "u'Romanian - Română': \"ro\", \n",
    "u'Russian - Русский': \"ru\", \n",
    "u'Russian - Русский (старая орфография)': \"ru-PETR1708\", \n",
    "u'Serbian - Српски': \"sr\", \n",
    "u'Serbian - Српски (латиница)': \"sr-Latn\", \n",
    "u'Slovak - Slovenčina': \"sk\", \n",
    "u'Slovenian - Slovenščina': \"sl\", \n",
    "u'Spanish - Español': \"es\", \n",
    "u'Swedish - Svenska': \"sv\", \n",
    "u'Tamil - தமிழ்': \"ta\", \n",
    "u'Telugu - తెలుగు': \"te\", \n",
    "u'Thai - ไทย': \"th\", \n",
    "u'Turkish - Türkçe': \"tr\", \n",
    "u'Ukrainian - Українська': \"uk\", \n",
    "u'Vietnamese - Tiếng Việt': \"vi\", \n",
    "u'Yiddish - Yiddish': \"yi\"}\n",
    "\n",
    "class Central(qt_utils.QMainWindow):\n",
    "    def __init__(self, parent=None):\n",
    "        super(Central, self).__init__(parent)\n",
    "        self.pushButton = QPushButton(\"OCR\")\n",
    "        self.pushButton.clicked.connect(self.on_pushButton_clicked)\n",
    "        self.dialogs = list()\n",
    "        self.combo = QComboBox(self)\n",
    "        self.populate_dict(self.combo)\n",
    "        self.codes = set(lang_hints_dict.values())\n",
    "        self.combo.activated.connect(self.handleActivated)\n",
    "        self.lang_hint = []    \n",
    "        self.multiButton = QPushButton(\"OCR Multiple languages\")\n",
    "        self.multiButton.clicked.connect(self.sd)\n",
    "        self.selected = []\n",
    "        layout = QVBoxLayout()\n",
    "        layout.addWidget(self.combo)\n",
    "        layout.addWidget(self.pushButton)\n",
    "        layout.addWidget(self.multiButton)\n",
    "        widget = QWidget()\n",
    "        widget.setLayout(layout)\n",
    "        widget.setFont(QFont('Arial', 11)) \n",
    "        self.setCentralWidget(widget)\n",
    "        self._gui_restore()\n",
    "    \n",
    "    def on_pushButton_clicked(self, mult = False):\n",
    "        if not mult:\n",
    "            self.lang_hint = [lang_hints_dict.get(str(self.combo.currentText()))]\n",
    "        print(\"button clicked\")\n",
    "        dialog = ocrWidget(self, self.lang_hint)\n",
    "        self.dialogs.append(dialog)\n",
    "        dialog.show()\n",
    "        self.setWindowOpacity(0.)\n",
    "    \n",
    "    def handleActivated(self, index):\n",
    "        self.lang_hint = [self.combo.itemData(index)]\n",
    "        \n",
    "    def populate_dict(self, combo : QComboBox):\n",
    "        for key, value in lang_hints_dict.items():\n",
    "            self.combo.addItem(key, value)\n",
    "            \n",
    "    def sd(self):\n",
    "        info_string = \"\"\"To enter multiple languages, enter the language code in a comma seperate list in order of importance. For example, \n",
    "    if you want to recognize a document with Arabic, English, and French in it, and (Arabic being most and French least important)\n",
    "    enter ar,en,fr. For the full list of language codes see https://cloud.google.com/vision/docs/languages#supported-langs\n",
    "    \"\"\"\n",
    "        text , ok = QInputDialog.getText(self,'InputDialog', info_string)\n",
    "        if ok:\n",
    "            lh = text.replace(\" \", \"\").split(',')\n",
    "            self.lang_hint.clear()\n",
    "            for elem in lh:\n",
    "                if elem not in self.codes:\n",
    "                    print(\"the language code {0} cannot be found! check your codes!\".format(elem))\n",
    "                    return\n",
    "                else:\n",
    "                    self.lang_hint.append(elem)\n",
    "            self.on_pushButton_clicked(True)\n",
    "\n",
    "    \n",
    "def main():\n",
    "    app = QtCore.QCoreApplication.instance()\n",
    "    if app is None:\n",
    "        app = QApplication(sys.argv)\n",
    "    main = Central()\n",
    "    main.show()\n",
    "    sys.exit(app.exec_())\n",
    "    \n",
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}