01_DataCollection_TheMet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "kBIEthy12B9Q",
    "outputId": "153fc3b7-5f6f-4e08-d077-4b791ad53b14"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mERROR: Could not find a version that satisfies the requirement chromium-chromedriver (from versions: none)\u001b[0m\n",
      "\u001b[31mERROR: No matching distribution found for chromium-chromedriver\u001b[0m\n",
      "cp: ./chromedriver: No such file or directory\n"
     ]
    }
   ],
   "source": [
    "# ! pip install selenium\n",
    "# ! pip install webcolors\n",
    "# ! pip install --upgrade google-cloud-bigquery\n",
    "\n",
    "# ! pip install validators\n",
    "\n",
    "# ! apt-get update\n",
    "! pip install chromium-chromedriver\n",
    "! cp ./chromedriver /usr/bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting bigquery\n",
      "  Downloading bigquery-0.0.20.tar.gz (7.7 kB)\n",
      "Collecting dbstream>=0.0.19\n",
      "  Downloading dbstream-0.1.10.tar.gz (6.3 kB)\n",
      "Requirement already satisfied: google-cloud-bigquery>=2.4.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from bigquery) (2.30.1)\n",
      "Collecting googleauthentication>=0.0.12\n",
      "  Downloading googleauthentication-0.0.16.tar.gz (3.8 kB)\n",
      "Collecting google-cloud-bigquery-storage>=2.1.0\n",
      "  Downloading google_cloud_bigquery_storage-2.10.0-py2.py3-none-any.whl (171 kB)\n",
      "\u001b[K     |████████████████████████████████| 171 kB 8.9 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pandas>=1.1.4\n",
      "  Downloading pandas-1.3.4-cp38-cp38-macosx_10_9_x86_64.whl (11.4 MB)\n",
      "\u001b[K     |████████████████████████████████| 11.4 MB 12.0 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pyarrow>=2.0.0\n",
      "  Downloading pyarrow-6.0.0-cp38-cp38-macosx_10_13_x86_64.whl (19.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 19.1 MB 9.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting sshtunnel==0.1.5\n",
      "  Downloading sshtunnel-0.1.5-py2.py3-none-any.whl (23 kB)\n",
      "Collecting dacktool>=0.0.7\n",
      "  Downloading dacktool-0.0.7.tar.gz (3.1 kB)\n",
      "Requirement already satisfied: requests>=2.22.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from dbstream>=0.0.19->bigquery) (2.24.0)\n",
      "Collecting google-api-core==2.1.1\n",
      "  Downloading google_api_core-2.1.1-py2.py3-none-any.whl (95 kB)\n",
      "\u001b[K     |████████████████████████████████| 95 kB 8.2 MB/s  eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-cloud-bigquery>=2.4.0->bigquery) (2.1.0)\n",
      "Requirement already satisfied: protobuf>=3.12.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-cloud-bigquery>=2.4.0->bigquery) (3.19.0)\n",
      "Requirement already satisfied: grpcio<2.0dev,>=1.38.1 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-cloud-bigquery>=2.4.0->bigquery) (1.41.0)\n",
      "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.4.1 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-cloud-bigquery>=2.4.0->bigquery) (2.2.0)\n",
      "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-cloud-bigquery>=2.4.0->bigquery) (2.8.1)\n",
      "Requirement already satisfied: proto-plus>=1.10.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-cloud-bigquery>=2.4.0->bigquery) (1.19.8)\n",
      "Requirement already satisfied: packaging>=14.3 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-cloud-bigquery>=2.4.0->bigquery) (21.0)\n",
      "Collecting google-api-python-client==1.7.11\n",
      "  Downloading google-api-python-client-1.7.11.tar.gz (142 kB)\n",
      "\u001b[K     |████████████████████████████████| 142 kB 11.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: google-auth>=1.23.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from googleauthentication>=0.0.12->bigquery) (2.3.0)\n",
      "Collecting google-auth-httplib2==0.0.3\n",
      "  Downloading google_auth_httplib2-0.0.3-py2.py3-none-any.whl (6.3 kB)\n",
      "Requirement already satisfied: google-auth-oauthlib>=0.4.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from googleauthentication>=0.0.12->bigquery) (0.4.6)\n",
      "Requirement already satisfied: cryptography>=2.7 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from googleauthentication>=0.0.12->bigquery) (2.9.2)\n",
      "Collecting google-cloud-secret-manager==2.7.2\n",
      "  Downloading google_cloud_secret_manager-2.7.2-py2.py3-none-any.whl (94 kB)\n",
      "\u001b[K     |████████████████████████████████| 94 kB 8.0 MB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting libcst>=0.2.5\n",
      "  Downloading libcst-0.3.21-py3-none-any.whl (514 kB)\n",
      "\u001b[K     |████████████████████████████████| 514 kB 10.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17.3; platform_machine != \"aarch64\" and platform_machine != \"arm64\" and python_version < \"3.10\" in /Users/doguma/anaconda3/lib/python3.8/site-packages (from pandas>=1.1.4->bigquery) (1.21.3)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from pandas>=1.1.4->bigquery) (2020.1)\n",
      "Collecting paramiko>=1.15.2\n",
      "  Downloading paramiko-2.8.0-py2.py3-none-any.whl (206 kB)\n",
      "\u001b[K     |████████████████████████████████| 206 kB 11.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: certifi>=2017.4.17 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from requests>=2.22.0->dbstream>=0.0.19->bigquery) (2020.6.20)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from requests>=2.22.0->dbstream>=0.0.19->bigquery) (2.10)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from requests>=2.22.0->dbstream>=0.0.19->bigquery) (1.25.11)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from requests>=2.22.0->dbstream>=0.0.19->bigquery) (3.0.4)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-api-core==2.1.1->dbstream>=0.0.19->bigquery) (1.53.0)\n",
      "Requirement already satisfied: setuptools>=40.3.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-api-core==2.1.1->dbstream>=0.0.19->bigquery) (58.2.0)\n",
      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery>=2.4.0->bigquery) (1.3.0)\n",
      "Requirement already satisfied: six>=1.5.2 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from grpcio<2.0dev,>=1.38.1->google-cloud-bigquery>=2.4.0->bigquery) (1.15.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from packaging>=14.3->google-cloud-bigquery>=2.4.0->bigquery) (2.4.7)\n",
      "Collecting httplib2<1dev,>=0.9.2\n",
      "  Downloading httplib2-0.20.2-py3-none-any.whl (96 kB)\n",
      "\u001b[K     |████████████████████████████████| 96 kB 9.9 MB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting uritemplate<4dev,>=3.0.0\n",
      "  Downloading uritemplate-3.0.1-py2.py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-auth>=1.23.0->googleauthentication>=0.0.12->bigquery) (4.7.2)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-auth>=1.23.0->googleauthentication>=0.0.12->bigquery) (4.2.4)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-auth>=1.23.0->googleauthentication>=0.0.12->bigquery) (0.2.8)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from google-auth-oauthlib>=0.4.0->googleauthentication>=0.0.12->bigquery) (1.3.0)\n",
      "Requirement already satisfied: cffi!=1.11.3,>=1.8 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from cryptography>=2.7->googleauthentication>=0.0.12->bigquery) (1.14.0)\n",
      "Collecting grpc-google-iam-v1<0.13dev,>=0.12.3\n",
      "  Downloading grpc-google-iam-v1-0.12.3.tar.gz (13 kB)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.2 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from libcst>=0.2.5->google-cloud-bigquery-storage>=2.1.0->bigquery) (3.7.4.3)\n",
      "Requirement already satisfied: pyyaml>=5.2 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from libcst>=0.2.5->google-cloud-bigquery-storage>=2.1.0->bigquery) (5.3.1)\n",
      "Collecting typing-inspect>=0.4.0\n",
      "  Downloading typing_inspect-0.7.1-py3-none-any.whl (8.4 kB)\n",
      "Collecting pynacl>=1.0.1\n",
      "  Downloading PyNaCl-1.4.0-cp35-abi3-macosx_10_10_x86_64.whl (380 kB)\n",
      "\u001b[K     |████████████████████████████████| 380 kB 5.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting bcrypt>=3.1.3\n",
      "  Downloading bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl (31 kB)\n",
      "Requirement already satisfied: pyasn1>=0.1.3 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from rsa<5,>=3.1.4->google-auth>=1.23.0->googleauthentication>=0.0.12->bigquery) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /Users/doguma/anaconda3/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib>=0.4.0->googleauthentication>=0.0.12->bigquery) (3.1.1)\n",
      "Requirement already satisfied: pycparser in /Users/doguma/anaconda3/lib/python3.8/site-packages (from cffi!=1.11.3,>=1.8->cryptography>=2.7->googleauthentication>=0.0.12->bigquery) (2.20)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting mypy-extensions>=0.3.0\n",
      "  Downloading mypy_extensions-0.4.3-py2.py3-none-any.whl (4.5 kB)\n",
      "Building wheels for collected packages: bigquery, dbstream, googleauthentication, dacktool, google-api-python-client, grpc-google-iam-v1\n",
      "  Building wheel for bigquery (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for bigquery: filename=bigquery-0.0.20-py3-none-any.whl size=9161 sha256=b03f1a816abc050c4c522dd7a90f37b61b7b5b194316986328b2c8baf3605087\n",
      "  Stored in directory: /Users/doguma/Library/Caches/pip/wheels/e1/99/21/a4b265bd46044ed58504686a85c2e787ba02cb7dfa7fb443e2\n",
      "  Building wheel for dbstream (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for dbstream: filename=dbstream-0.1.10-py3-none-any.whl size=7502 sha256=579d9854f90a3e556145aa72ab51e5ffbf8289b61f0d6b2817029fae3fe67de7\n",
      "  Stored in directory: /Users/doguma/Library/Caches/pip/wheels/5b/5a/39/c988bd861bbcff0deec6086dd1f413f339c88d18e80c7d767e\n",
      "  Building wheel for googleauthentication (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for googleauthentication: filename=googleauthentication-0.0.16-py3-none-any.whl size=4735 sha256=d6bbfb8a8df57115cc18fab29b5678602722840950a882ccdb0455e6f589016a\n",
      "  Stored in directory: /Users/doguma/Library/Caches/pip/wheels/8a/c3/b1/37190b8ddc140af443c19428a896c65e47f0f53aa32e2875a4\n",
      "  Building wheel for dacktool (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for dacktool: filename=dacktool-0.0.7-py3-none-any.whl size=4802 sha256=39f403bf70765ebaac5416ed76c8c3727bed97f70b35edb6a5f0450a103c5b62\n",
      "  Stored in directory: /Users/doguma/Library/Caches/pip/wheels/f2/a7/a2/692f34fd9af96492bb49a0dbb65abe96bb0e9da15eb4df99f9\n",
      "  Building wheel for google-api-python-client (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for google-api-python-client: filename=google_api_python_client-1.7.11-py3-none-any.whl size=56543 sha256=f18d76d25fea5fac63211f613688db42bd738b425175f26603007c320b455bbb\n",
      "  Stored in directory: /Users/doguma/Library/Caches/pip/wheels/69/69/aa/24c58209ab280c154bb17a8ab37294226d776a5cc86aea03b4\n",
      "  Building wheel for grpc-google-iam-v1 (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for grpc-google-iam-v1: filename=grpc_google_iam_v1-0.12.3-py3-none-any.whl size=18515 sha256=58786a66b6feba190997f2f7114b713661f8c2d3727674b94e3c9247769f945a\n",
      "  Stored in directory: /Users/doguma/Library/Caches/pip/wheels/8f/b9/13/fce3d62261f63c01b28281fe6a9d704a7af65d96ff2c88552e\n",
      "Successfully built bigquery dbstream googleauthentication dacktool google-api-python-client grpc-google-iam-v1\n",
      "\u001b[31mERROR: phik 0.12.0 has requirement scipy>=1.5.2, but you'll have scipy 1.5.0 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: pdpbox 0.2.1 has requirement matplotlib==3.1.1, but you'll have matplotlib 3.4.3 which is incompatible.\u001b[0m\n",
      "Installing collected packages: pynacl, bcrypt, paramiko, sshtunnel, dacktool, google-api-core, dbstream, httplib2, google-auth-httplib2, uritemplate, google-api-python-client, grpc-google-iam-v1, mypy-extensions, typing-inspect, libcst, google-cloud-secret-manager, googleauthentication, google-cloud-bigquery-storage, pandas, pyarrow, bigquery\n",
      "  Attempting uninstall: google-api-core\n",
      "    Found existing installation: google-api-core 2.2.2\n",
      "    Uninstalling google-api-core-2.2.2:\n",
      "      Successfully uninstalled google-api-core-2.2.2\n",
      "  Attempting uninstall: pandas\n",
      "    Found existing installation: pandas 1.0.5\n",
      "    Uninstalling pandas-1.0.5:\n",
      "      Successfully uninstalled pandas-1.0.5\n",
      "Successfully installed bcrypt-3.2.0 bigquery-0.0.20 dacktool-0.0.7 dbstream-0.1.10 google-api-core-2.1.1 google-api-python-client-1.7.11 google-auth-httplib2-0.0.3 google-cloud-bigquery-storage-2.10.0 google-cloud-secret-manager-2.7.2 googleauthentication-0.0.16 grpc-google-iam-v1-0.12.3 httplib2-0.20.2 libcst-0.3.21 mypy-extensions-0.4.3 pandas-1.3.4 paramiko-2.8.0 pyarrow-6.0.0 pynacl-1.4.0 sshtunnel-0.1.5 typing-inspect-0.7.1 uritemplate-3.0.1\n"
     ]
    }
   ],
   "source": [
    "! pip install bigquery"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "CRBaJTsWLUjB"
   },
   "outputs": [],
   "source": [
    "import os, time, sys, io\n",
    "import cv2, keras, glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from collections import Counter\n",
    "from numpy import asarray\n",
    "from io import BytesIO\n",
    "\n",
    "from google.oauth2 import service_account\n",
    "from google.cloud import bigquery\n",
    "\n",
    "import requests\n",
    "import PIL\n",
    "import re\n",
    "import urllib\n",
    "import urllib.request\n",
    "import webcolors\n",
    "import validators\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from bs4 import BeautifulSoup\n",
    "from PIL import Image, ImageColor\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cDIr-JKlaK0F",
    "outputId": "c0aabf3d-7f3f-43c9-88f8-864befc051cb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "HLqlV0w111TO"
   },
   "outputs": [],
   "source": [
    "def start_chromedriver():\n",
    "    sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')\n",
    "    chrome_options = webdriver.ChromeOptions()\n",
    "    chrome_options.add_argument('--headless')\n",
    "    chrome_options.add_argument('--no-sandbox')\n",
    "    chrome_options.add_argument('--disable-dev-shm-usage')\n",
    "\n",
    "    driver = webdriver.Chrome('chromedriver', options=chrome_options)\n",
    "    \n",
    "    return driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "NSQ-T3HAOqAW"
   },
   "outputs": [],
   "source": [
    "# Hex to RGB Conversion\n",
    "# Source: https://gist.github.com/kb22/f17e59a79d4fcca02188c23cca932be5#file-rgb2hex-py\n",
    "\n",
    "def hex2name(c):\n",
    "    h_color = '#{:02x}{:02x}{:02x}'.format(int(c[0]), int(c[1]), int(c[2]))\n",
    "\n",
    "    rms_lst = []\n",
    "    for img_clr, img_hex in webcolors.CSS3_NAMES_TO_HEX.items():\n",
    "        cur_clr = webcolors.hex_to_rgb(img_hex)\n",
    "        rmse = np.sqrt(mean_squared_error(c, cur_clr))\n",
    "        rms_lst.append(rmse)\n",
    "\n",
    "    closest_color = rms_lst.index(min(rms_lst))\n",
    "\n",
    "    nm = list(webcolors.CSS3_NAMES_TO_HEX.items())[closest_color][0]\n",
    "\n",
    "    return nm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "WhP_tHNV82wu"
   },
   "outputs": [],
   "source": [
    "# connecting to google cloud platform\n",
    "\n",
    "! gcloud iam service-accounts create dopoem\n",
    "! gcloud projects add-iam-policy-binding dopoem --member=\"serviceAccount:dopoem@dopoem.iam.gserviceaccount.com\" --role=\"roles/owner\"\n",
    "! gcloud auth login"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "_I3cm9s_Sf-B"
   },
   "outputs": [],
   "source": [
    "! export GOOGLE_APPLICATION_CREDENTIALS=\"/home/user/Downloads\"\n",
    "! export GOOGLE_APPLICATION_CREDENTIALS=\"/home/user/Downloads/service-account-file.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "O-16-HixVxiz"
   },
   "outputs": [],
   "source": [
    "# get json file from google cloud platform\n",
    "\n",
    "credentials = \"dopoem-f5aa6b994716.json\"\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "SSckqhimSAKc"
   },
   "outputs": [],
   "source": [
    "# Google Cloud's client to pull data\n",
    "\n",
    "client = bigquery.Client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Querying the_met.objects data from \"bigquery-public-data\"\n",
    "\n",
    "QUERY5 = (\"\"\"\n",
    "SELECT object_id, LOWER(department), LOWER(object_name), \\\n",
    "        LOWER(culture), object_begin_date\n",
    "        \n",
    "FROM bigquery-public-data.the_met.objects,\n",
    "UNNEST(SPLIT(culture, ',')) culture,\n",
    "UNNEST(SPLIT(object_name, ',')) object_name,\n",
    "UNNEST(SPLIT(department, ',')) department\n",
    "\n",
    "LIMIT 400\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_job5 = client.query(QUERY5)\n",
    "rows5 = query_job5.result()\n",
    "\n",
    "oid, o_type, o_country, o_date = [], [], [], []\n",
    "\n",
    "for i in rows5:\n",
    "    oid.append(i[0])\n",
    "    o_type.append(i[2])\n",
    "    o_country.append(i[3])\n",
    "    o_date.append(i[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oid</th>\n",
       "      <th>o_type</th>\n",
       "      <th>o_country</th>\n",
       "      <th>o_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>38568</td>\n",
       "      <td>plaque</td>\n",
       "      <td>tibet</td>\n",
       "      <td>1500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>41900</td>\n",
       "      <td>plaque</td>\n",
       "      <td>afghanistan</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41900</td>\n",
       "      <td>plaque</td>\n",
       "      <td>possibly of west indian manufacture</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>60517</td>\n",
       "      <td>pipe case</td>\n",
       "      <td>japan</td>\n",
       "      <td>1800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60518</td>\n",
       "      <td>pipe case</td>\n",
       "      <td>japan</td>\n",
       "      <td>1800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     oid     o_type                             o_country  o_date\n",
       "0  38568     plaque                                 tibet    1500\n",
       "1  41900     plaque                           afghanistan       1\n",
       "2  41900     plaque   possibly of west indian manufacture       1\n",
       "3  60517  pipe case                                 japan    1800\n",
       "4  60518  pipe case                                 japan    1800"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "met_objects = pd.DataFrame()\n",
    "\n",
    "met_objects['oid'], met_objects['o_type'], met_objects['o_country'], \\\n",
    "met_objects['o_date'] = oid, o_type, o_country, o_date\n",
    "\n",
    "met_objects.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(71, 25, 32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(met_objects['o_type'].unique()), len(met_objects['o_country'].unique()),\\\n",
    "len(met_objects['o_date'].unique())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['tibet', 'afghanistan', ' possibly of west indian manufacture',\n",
       "       'japan', 'tibet (or nepal)', 'burma', 'nepal', 'northwest china',\n",
       "       'china', 'india', 'pakistan (ancient region of gandhara)',\n",
       "       'india (jammu and kashmir', ' ancient kingdom of kashmir)', '',\n",
       "       'sri lanka', 'india (orissa?)', 'india (orissa)',\n",
       "       'india (andhra pradesh or west bengal)', 'india (karnataka',\n",
       "       ' mysore or tamil nadu)', ' indo-portuguese (goa or diu)',\n",
       "       'south india',\n",
       "       'pakistan (ancient region of gandhara) or afghanistan',\n",
       "       'india (bengal)', 'thailand'], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "met_objects['o_country'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "countries_crop = []\n",
    "\n",
    "for i in met_objects['o_country']:\n",
    "    temp = i\n",
    "    temp = re.sub('\\)', '', temp)\n",
    "    \n",
    "    if 'south india' in temp:\n",
    "        temp = re.sub(' ', '-', temp)\n",
    "        countries_crop.append(temp)\n",
    "    elif 'india' in temp or 'indian' in temp:\n",
    "        countries_crop.append('india')\n",
    "    elif temp == '':\n",
    "        countries_crop.append('others')\n",
    "    elif temp.find('(')+1:\n",
    "        temp = temp[:temp.find('(')].strip()\n",
    "        temp = re.sub(' ', '-', temp)\n",
    "        countries_crop.append(temp)\n",
    "    else:\n",
    "        temp = temp.strip()\n",
    "        temp = re.sub(' ', '-', temp)\n",
    "        countries_crop.append(temp)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'afghanistan',\n",
       " 'ancient-kingdom-of-kashmir',\n",
       " 'burma',\n",
       " 'china',\n",
       " 'india',\n",
       " 'indo-portuguese',\n",
       " 'japan',\n",
       " 'mysore-or-tamil-nadu',\n",
       " 'nepal',\n",
       " 'northwest-china',\n",
       " 'others',\n",
       " 'pakistan',\n",
       " 'south-india',\n",
       " 'sri-lanka',\n",
       " 'thailand',\n",
       " 'tibet'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(countries_crop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "met_objects['o_country'] = countries_crop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['tibet', 'afghanistan', 'india', 'japan', 'burma', 'nepal',\n",
       "       'northwest-china', 'china', 'pakistan',\n",
       "       'ancient-kingdom-of-kashmir', 'others', 'sri-lanka',\n",
       "       'mysore-or-tamil-nadu', 'indo-portuguese', 'south-india',\n",
       "       'thailand'], dtype=object)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "met_objects.dropna(inplace=True)\n",
    "met_objects['o_country'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(met_objects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "! mkdir the_met_objects/train the_met_objects/test\n",
    "\n",
    "for i in met_objects['o_country'].unique():\n",
    "    ! mkdir the_met_objects/train/{i}\n",
    "    \n",
    "for i in met_objects['o_country'].unique():\n",
    "    ! mkdir the_met_objects/test/{i}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_url(object_id):\n",
    "    driver = start_chromedriver()\n",
    "    driver.get('https://www.metmuseum.org/art/collection/search/' + str(object_id))\n",
    "    html = driver.page_source\n",
    "\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "    li_tag =  soup.find('a', 'gtm__download__image')\n",
    "    href = li_tag['href']\n",
    "    \n",
    "\n",
    "    driver.quit()\n",
    "    \n",
    "    return href"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def url_to_image(link):\n",
    "  response = requests.get(link)\n",
    "  # get image links that only respond with status code of 200\n",
    "  # ignore the 'not found' pages\n",
    "  if response.status_code == 200:\n",
    "    image_bytes = io.BytesIO(response.content)\n",
    "    img = PIL.Image.open(image_bytes)\n",
    "    return img\n",
    "  else:\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/doguma/anaconda3/lib/python3.8/site-packages/PIL/Image.py:2918: DecompressionBombWarning: Image size (100920000 pixels) exceeds limit of 89478485 pixels, could be decompression bomb DOS attack.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "remove_indexes = []\n",
    "\n",
    "for index, el in met_objects[:320].iterrows():\n",
    "    image_link = get_url(el['oid'])\n",
    "    image = url_to_image(image_link)\n",
    "    \n",
    "    if image != None:\n",
    "        image.save(r'the_met_objects/train/'+ str(el['o_country'])+\\\n",
    "                   '/' + str(el['oid'])+'.png')\n",
    "    else:\n",
    "        remove_indexes.append(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove_indexes2 = []\n",
    "\n",
    "for index, el in met_objects[320:].iterrows():\n",
    "    image_link = get_url(el['oid'])\n",
    "    image = url_to_image(image_link)\n",
    "    \n",
    "    if image != None:\n",
    "        image.save(r'the_met_objects/test/'+ str(el['o_country'])+\\\n",
    "                   '/' + str(el['oid'])+'.png')\n",
    "    else:\n",
    "        remove_indexes2.append(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove_indexes, remove_indexes2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# met_objects.to_csv('met_objects')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "dopoem.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}