imageSelection/imageSelection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from extractors import extractors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def filter_images(images):\n",
    "    print('Initial Images: {}'.format(len(images)))\n",
    "    images = images.drop_duplicates(['detailsUrl'])\n",
    "    images = images.drop_duplicates(['imageUrl'])\n",
    "    images = images.drop_duplicates(['title'])\n",
    "    print('After dropping duplicates: {}'.format(len(images)))\n",
    "    images = images[images.width >= images.height]\n",
    "    print('After filtering horizontal images: {}'.format(len(images)))\n",
    "    images = images[images.imageUrl.str.contains('FRAME') == False]\n",
    "    print('After filtering empty frames: {}'.format(len(images)))\n",
    "    return images\n",
    "    \n",
    "\n",
    "def export_images(images, filename):\n",
    "    export_images = images\n",
    "    export_images = export_images.drop('height', 1)\n",
    "    export_images = export_images.drop('width', 1)\n",
    "    export_images.completionYear = export_images.completionYear.fillna(-1).astype(int)\n",
    "    export_images.imageUrl = export_images.imageUrl.str.replace('!Large.jpg', '')\n",
    "    export_images.to_csv(filename, sep=',', encoding='utf-8', index=False)\n",
    "    print('Exported {} images to \"{}\"'.format(len(export_images), filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetched 1198 images from muzei archive\n",
      "Error fetching details page http://www.wikiart.org/en/amadeo-de-souza-cardoso/sem-t-tulo-clown-cavalo-salamandra-1912?utm_source=Muzei&utm_campaign=Muzei: HTTP Error 404: Not Found\n",
      "Error fetching details page http://www.wikiart.org/de/albert-trachsel/the-island-of-blossoming-trees?utm_source=Muzei&utm_campaign=Muzei: HTTP Error 404: Not Found\n",
      "Error fetching details page http://www.wikiart.org/de/albert-trachsel/droomlandschap-1905?utm_source=Muzei&utm_campaign=Muzei: HTTP Error 404: Not Found\n",
      "Error fetching details page http://www.wikiart.org/en/amadeo-de-souza-cardoso/sem-t-tulo-clown-cavalo-salamandra-1912?utm_source=Muzei&utm_campaign=Muzei?utm_source=Muzei&utm_campaign=Muzei: HTTP Error 404: Not Found\n",
      "Muzei images: 1194\n",
      "Error getting details url for image https://uploads8.wikiart.org/vertumnus-emperor-rudolph-ii(1).jpg!Large.jpg\n",
      "Most viewed images: 600\n",
      "Initial Images: 1794\n",
      "After dropping duplicates: 805\n",
      "After filtering horizontal images: 471\n",
      "After filtering empty frames: 470\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artistName</th>\n",
       "      <th>completionYear</th>\n",
       "      <th>detailsUrl</th>\n",
       "      <th>height</th>\n",
       "      <th>imageUrl</th>\n",
       "      <th>title</th>\n",
       "      <th>width</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Vincent van Gogh</td>\n",
       "      <td>1889</td>\n",
       "      <td>http://www.wikipaintings.org/en/vincent-van-go...</td>\n",
       "      <td>594</td>\n",
       "      <td>https://uploads4.wikiart.org/00142/images/vinc...</td>\n",
       "      <td>The Starry Night</td>\n",
       "      <td>750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Georges Seurat</td>\n",
       "      <td>1886</td>\n",
       "      <td>http://www.wikipaintings.org/en/georges-seurat...</td>\n",
       "      <td>507</td>\n",
       "      <td>https://uploads4.wikiart.org/images/georges-se...</td>\n",
       "      <td>Sunday Afternoon on the Island of La Grande Jatte</td>\n",
       "      <td>750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Claude Monet</td>\n",
       "      <td>1884</td>\n",
       "      <td>http://www.wikipaintings.org/en/claude-monet/h...</td>\n",
       "      <td>598</td>\n",
       "      <td>https://uploads4.wikiart.org/images/claude-mon...</td>\n",
       "      <td>Haystacks at Giverny</td>\n",
       "      <td>750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         artistName completionYear  \\\n",
       "0  Vincent van Gogh           1889   \n",
       "1    Georges Seurat           1886   \n",
       "3      Claude Monet           1884   \n",
       "\n",
       "                                          detailsUrl height  \\\n",
       "0  http://www.wikipaintings.org/en/vincent-van-go...    594   \n",
       "1  http://www.wikipaintings.org/en/georges-seurat...    507   \n",
       "3  http://www.wikipaintings.org/en/claude-monet/h...    598   \n",
       "\n",
       "                                            imageUrl  \\\n",
       "0  https://uploads4.wikiart.org/00142/images/vinc...   \n",
       "1  https://uploads4.wikiart.org/images/georges-se...   \n",
       "3  https://uploads4.wikiart.org/images/claude-mon...   \n",
       "\n",
       "                                               title width  \n",
       "0                                   The Starry Night   750  \n",
       "1  Sunday Afternoon on the Island of La Grande Jatte   750  \n",
       "3                               Haystacks at Giverny   750  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "most_viewed_extractor = extractors.MostViewedWikiArtExtractor()\n",
    "muzei_extractor = extractors.MuzeiExtractor()\n",
    "\n",
    "muzei_images = muzei_extractor.extract_images()\n",
    "print('Muzei images: {}'.format(len(muzei_images)))\n",
    "most_viewed_images = most_viewed_extractor.extract_images()\n",
    "print('Most viewed images: {}'.format(len(most_viewed_images)))\n",
    "images = pd.DataFrame.from_records([i.to_dict() for i in muzei_images + most_viewed_images])\n",
    "\n",
    "images = filter_images(images)\n",
    "images.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exported 470 images to \"../server/data/images.csv\"\n"
     ]
    }
   ],
   "source": [
    "FILENAME = '../server/data/images.csv'\n",
    "export_images(images, FILENAME)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}