image_features_dask.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dask.array.image import imread\n",
    "import cv2\n",
    "from dask.diagnostics import ProgressBar\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def auto_canny_sub(image, sigma=0.33):\n",
    "    # compute the median of the single channel pixel intensities\n",
    "    v = np.median(image)\n",
    " \n",
    "    # apply automatic Canny edge detection using the computed median\n",
    "    lower = int(max(0, (1.0 - sigma) * v))\n",
    "    upper = int(min(255, (1.0 + sigma) * v))\n",
    "    edged = cv2.Canny(image, lower, upper)\n",
    " \n",
    "    # return the edged image\n",
    "    return edged\n",
    "\n",
    "def auto_canny(x):\n",
    "    gray = cv2.cvtColor(x, cv2.COLOR_BGR2GRAY)\n",
    "    blurred = cv2.GaussianBlur(gray, (3, 3), 0)\n",
    "\n",
    "    # apply Canny edge detection using a wide threshold, tight\n",
    "    # threshold, and automatically determined threshold\n",
    "    auto = auto_canny_sub(blurred)\n",
    "    result = (float(np.sum(auto)) / (x.shape[0]*x.shape[1]))\n",
    "    return result\n",
    "\n",
    "def get_features(x):\n",
    "    x = x[0, :, :]\n",
    "    a_canny = auto_canny(x)\n",
    "    #b_score = get_blurrness_score(x)\n",
    "    #avg_color = get_average_color(x)\n",
    "    #w_h = get_width_height(x)\n",
    "    return np.array([[[[a_canny]]]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read images. dask.array<imread, shape=(734, 360, 480, 3), dtype=uint8, chunksize=(1, 360, 480, 3)>\n",
      "[########################################] | 100% Completed |  1.3s\n"
     ]
    }
   ],
   "source": [
    "#IMAGE_DIR = '/home/ubuntu/.kaggle/competitions/avito-demand-prediction/test_jpg/data/competition_files/test_jpg/'\n",
    "IMAGE_DIR = '/Users/Rashmi/Downloads/sample_trainimages1/'\n",
    "im = imread( IMAGE_DIR + '*.jpg', imread=cv2.imread)  \n",
    "print(\"Read images.\",im)\n",
    "result = im.map_blocks(get_features, dtype=im.dtype, chunks=(3500,1,1,1))\n",
    "\n",
    "s = None\n",
    "with ProgressBar():\n",
    "    s = result.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([31.23159722, 13.33289931, 12.22612847, 16.2296875 , 24.48      ,\n",
       "        8.46851852, 49.85486111, 45.77050781,  5.55844907,  8.38637153,\n",
       "       21.93628976, 33.57762346, 51.87635659, 13.67673611,  2.10434028,\n",
       "       26.8546875 ,  9.37213542, 19.82005208,  6.25694444, 14.09420668,\n",
       "       36.40169271,  7.57364206, 11.33185764, 18.65933642, 10.59445193,\n",
       "        3.09305556, 10.19409722, 11.94722222,  6.84131944, 16.4431713 ,\n",
       "        8.5299048 ,  3.64083333, 24.72821181,  5.75520833, 64.121875  ,\n",
       "        1.47569444,  0.77473958,  6.96921296,  3.88107639, 18.76197917,\n",
       "       14.97792969, 12.42682292, 15.42543403, 18.22777778, 13.33880208,\n",
       "       33.50416667, 17.97696274,  7.06267361, 21.60613426,  7.04825079,\n",
       "       51.71172386, 12.09589844, 14.44852431,  5.46449653, 12.24088542,\n",
       "       22.86883681,  8.71840278, 27.56892361, 20.09754393,  4.98046875,\n",
       "       12.42337963, 20.01272502,  6.58553241, 36.28240741, 51.51293103,\n",
       "        6.95606557,  3.87960069, 12.49618056, 43.87337963, 14.03680556,\n",
       "       23.89444444, 33.42731546,  6.05329861,  7.99309896,  2.84071181,\n",
       "       24.53342014, 25.71298368, 14.37916667,  8.86597222, 13.07006173,\n",
       "       34.7248062 ,  5.33758681, 25.26388889,  9.16111111, 18.07802714,\n",
       "        8.39965278, 10.85284722, 33.13998614, 24.74277286,  7.7319812 ,\n",
       "        5.33758681, 27.49809028, 38.96207068,  5.46302083, 13.40815972,\n",
       "        4.14768519,  9.68498264, 30.76724537, 36.53151441,  5.6991693 ,\n",
       "       20.98955432,  9.89305556,  5.94114583, 50.2764881 , 19.77627315,\n",
       "        5.45711806,  4.15358796, 12.49618056, 33.3422619 , 15.41621094,\n",
       "       11.89114583, 18.84363426, 21.32059801, 18.68975904, 18.39305556,\n",
       "       17.75782387, 25.4047619 , 33.02013889, 12.79574653, 15.68663194,\n",
       "       17.56519097, 26.37213542, 15.94930556, 28.36833333, 12.14053819,\n",
       "       14.56067708,  6.19644097, 30.78888889, 19.73298611,  7.59539931,\n",
       "       26.87387153, 29.51241319, 50.39939236,  9.34016204,  5.30954861,\n",
       "       14.84548611, 21.00355903, 46.97688802, 20.30703125,  5.92638889,\n",
       "       26.32638889, 21.21753472, 12.91232639,  7.065625  , 26.83919271,\n",
       "        3.31957053, 11.38203125,  5.46990741, 14.03287037,  3.12132353,\n",
       "       41.54375   , 19.54778646,  4.01684028, 47.45      ,  9.80813172,\n",
       "        7.67980519,  9.65251736,  6.76015625, 13.65164931, 19.21796875,\n",
       "        5.14487416,  3.01828704,  5.80390625,  8.553125  , 35.52291667,\n",
       "       17.32170139, 14.63298611, 39.72421875,  7.45520833, 21.37592593,\n",
       "       10.09670139, 11.89881506, 11.62699653, 11.20478395, 10.41397569,\n",
       "       10.11295181, 26.84583333, 24.5181713 , 19.3698398 , 12.8833682 ,\n",
       "       13.02724609,  6.53585069, 16.69010417, 18.13153249, 31.38506944,\n",
       "       16.42349537, 36.12064549, 11.86163194, 10.04652778, 33.74470486,\n",
       "       32.73237847, 34.47074653, 25.45476472, 12.41059028,  3.18370807,\n",
       "       19.24556738,  5.81220703, 12.90789931, 12.71310764, 22.69433594,\n",
       "       11.1828125 ,  5.04835069, 31.14157986, 10.35494792, 10.29628906,\n",
       "       39.09037456, 23.75517191,  6.12511574, 12.08446181, 10.82569444,\n",
       "       10.80601852, 13.2753752 , 19.18009259,  3.94453125, 12.7334308 ,\n",
       "       25.57689145, 31.87647569, 23.11232639,  9.55659722,  5.87964527,\n",
       "       26.21866319,  3.83828125,  8.71734442, 17.46742491,  9.721875  ,\n",
       "       11.53255208,  6.02083333, 16.36692708,  2.10729167, 10.82766204,\n",
       "       13.03333333, 10.23312457, 28.25757576, 24.20581597, 22.58550347,\n",
       "       25.10038337, 16.57795139,  5.45023148, 17.63897569, 12.24236111,\n",
       "       32.82239583,  4.90373264, 24.26705729, 10.2806713 , 14.82714675,\n",
       "        9.78828125, 31.86467014, 29.25657362, 37.49592014, 19.27256944,\n",
       "        9.15981735, 13.6796875 , 16.36446759, 13.70034722, 15.55381944,\n",
       "        6.24108073, 14.64921875, 24.47291667,  5.6966404 , 18.91053241,\n",
       "       17.28776042, 15.0078125 , 17.14609375, 14.77465278,  2.60755208,\n",
       "       17.66996528,  2.24453125, 34.42057292,  9.03125   , 17.23315972,\n",
       "       40.71440972,  2.97795139, 33.18976955, 12.08151042, 28.37972786,\n",
       "        5.55844907,  5.66371528, 17.70243056,  7.66623264,  2.62230903,\n",
       "       15.34427083,  8.37899306, 33.27986111, 22.98197492,  2.05416667,\n",
       "       12.14398148, 48.19618056, 13.67378472, 20.92009132,  8.29487847,\n",
       "       21.1395    , 20.36605903,  4.74507168,  6.89739583, 25.69184028,\n",
       "        9.26736111, 22.06064815,  6.44878472, 37.14959724, 16.11285689,\n",
       "       25.56770108,  4.03159722,  8.33275463, 21.44036458, 15.05798611,\n",
       "       36.05121528,  2.44110202, 30.02300347,  3.05911458, 11.66979167,\n",
       "       18.62326389, 19.24892572, 12.08888889, 16.04422222, 17.29071181,\n",
       "        4.15703125, 18.02929688,  9.09175347, 17.17154948,  2.59958333,\n",
       "        4.43298611, 22.17378472, 10.4921875 , 15.36341757, 11.82473958,\n",
       "       18.29766667, 12.73229167, 22.99869792, 30.5670816 ,  4.45069444,\n",
       "       15.51754539, 18.60260417, 12.71886447,  8.5265625 , 21.29490457,\n",
       "       31.838875  , 10.35397412, 15.02190992, 18.32502857, 12.03641975,\n",
       "        4.39314236,  6.19976128, 11.51927083, 22.21018519, 11.41479857,\n",
       "        6.03522135, 16.11414392,  3.76962857, 16.62525934,  7.16203704,\n",
       "        8.45388455,  0.89131944, 37.47673611, 10.48480903,  2.60034952,\n",
       "       25.26213992,  6.8893643 ,  8.64035494, 19.18845486, 18.44913194,\n",
       "       15.93454861, 32.31577885,  2.51721429,  3.57560764,  7.38142361,\n",
       "       14.72300347, 10.10481771,  8.40260417,  1.98554688,  9.28805351,\n",
       "        8.25798611, 23.04296875, 12.28164845, 19.91305351, 24.74887153,\n",
       "       17.58831019, 39.4462963 , 11.56649306, 24.74444444, 30.98958333,\n",
       "       10.23836806, 28.52615741, 24.87824074,  3.42803819, 17.33580247,\n",
       "        2.55590278, 13.24780093, 30.09826389,  8.23105469, 10.24574653,\n",
       "       14.85286458,  3.16684028, 25.38932292, 25.51967593, 12.44305556,\n",
       "       19.62526042,  9.99635417, 10.52268519, 29.18333333,  9.81631944,\n",
       "       19.01137153, 10.24214385,  9.89502315,  5.39366319, 27.44053819,\n",
       "        4.55694444,  2.46588542, 17.18003472,  7.29386574, 31.503125  ,\n",
       "        9.58795573, 18.5115894 ,  9.13356481, 30.36962861,  9.32934028,\n",
       "       15.66302083, 11.44266304, 20.26030093, 17.19331597, 14.74071181,\n",
       "       13.6275463 ,  9.21866319, 35.57456597, 32.14144941,  3.42803819,\n",
       "       10.22518519, 36.35373264,  8.15763889, 59.12418981, 52.66310764,\n",
       "       31.43671875, 19.18402778, 22.76617375, 17.34727626, 22.06064815,\n",
       "       18.39895833, 23.49895833, 30.5193287 , 23.109375  ,  4.55546875,\n",
       "       17.88174274,  9.62546296,  2.01432292,  3.73793403,  5.65486111,\n",
       "       16.328594  , 19.96393229,  4.21605903,  8.61215278,  9.67465278,\n",
       "       10.87685185, 23.1349537 ,  7.88315972, 10.64713542,  9.00911458,\n",
       "        6.44583333,  3.01189236, 31.00729167,  5.54270833,  6.55601852,\n",
       "       33.45673077, 14.39576823,  3.28932292, 18.43658854, 11.32152778,\n",
       "       28.77604167, 30.25763889, 17.29956597, 13.82249216,  5.11328125,\n",
       "        9.83255208, 22.51725985, 14.13912037,  4.36067708, 11.79899691,\n",
       "       25.24175347, 32.04153605, 25.76119792, 44.93268229,  7.16154514,\n",
       "        4.66240741, 16.89768519, 10.22065972, 36.53007813, 30.22025463,\n",
       "       21.07291667,  7.56153305, 11.55099826, 41.98402141, 12.71032158,\n",
       "        0.61979167, 28.89704861, 40.26432292, 19.11614583, 18.68671875,\n",
       "        8.58814815,  6.90182292, 17.02877604,  8.7790404 , 14.63298611,\n",
       "        3.54560185, 18.70262809,  6.85755208, 15.19965278, 18.84363426,\n",
       "       20.68923611, 11.71884218, 30.47604167, 14.48285167, 18.68819444,\n",
       "       16.8729525 ,  6.68489583, 19.62526042, 34.80572917, 50.95277778,\n",
       "       20.70402033, 10.90076336, 17.60798611,  8.11041667,  6.78819444,\n",
       "       40.65439815,  2.3375    ,  1.16875   , 29.51301602,  6.83984375,\n",
       "        8.24027778, 17.44713542,  7.88758681, 28.85365226, 16.25182292,\n",
       "       10.14835069, 16.48645833,  2.07630208, 10.86295573,  5.59583333,\n",
       "       12.49470486,  8.21371528,  5.64807524, 13.94826389,  5.81571181,\n",
       "       25.55460069,  8.45130208, 15.54201389,  4.42118056, 35.4068287 ,\n",
       "        9.1375    , 12.93630642,  2.05841155, 31.93219462, 17.10514323,\n",
       "        2.66731771,  4.21753472,  9.9320007 , 15.63409338, 22.44547708,\n",
       "       31.18289931, 19.6306713 ,  1.21356033, 26.51232639, 28.68307292,\n",
       "       13.57454427, 27.7037037 ,  4.83808576, 21.36362847,  8.48081597,\n",
       "       13.44510719,  2.06006944, 21.42413194,  5.48496377, 14.69939236,\n",
       "        8.50110677, 24.21983507,  5.83194444, 24.384375  , 33.78251953,\n",
       "       11.6875    , 13.87372212, 56.40891204, 12.96840278,  6.41484375,\n",
       "       16.26412037, 16.51030655, 17.64782986, 12.84887153,  7.51004464,\n",
       "       13.94199219,  5.32744141, 13.37569444,  9.98946759, 12.74016204,\n",
       "       16.47170139, 23.89444444,  1.07467448, 42.67511574, 14.10960648,\n",
       "       36.5755787 ,  7.08886719, 16.98559766, 45.11973392,  4.64548611,\n",
       "       11.21085069,  3.40609319,  4.58449074,  7.98645833, 20.15060764,\n",
       "       26.5900463 , 15.96111111, 15.82799443,  2.63559028,  4.89782986,\n",
       "       27.56006944, 19.96319444,  7.13424479, 11.8203125 , 26.85526316,\n",
       "        7.45373264, 10.63680556,  4.33854167, 53.26113281, 10.98310185,\n",
       "       43.78680556, 62.59157986,  3.66414931,  9.9266276 , 43.89453125,\n",
       "       15.6453125 ,  4.57594531, 21.36805556,  7.40355903, 23.88263889,\n",
       "        6.48518519, 12.27462963,  2.58873698,  8.659375  ,  5.74045139,\n",
       "       19.87662037,  9.01565199, 17.9775    , 11.75538194,  5.50286458,\n",
       "       22.27383552, 35.54505208,  8.37456597, 12.72520833, 21.47725694,\n",
       "       22.64934937, 20.95954438,  9.15665236,  6.73838976, 35.82723765,\n",
       "       25.52335165,  7.58506944, 37.03255208,  9.04010417, 16.50121528,\n",
       "       16.53660436, 37.33443245, 17.77473958,  5.10655864, 16.15147569,\n",
       "        7.98793403, 10.25755208,  9.71597222,  7.43602431, 36.56770833,\n",
       "        9.10355903,  7.15157807,  7.23229261, 26.28310185,  8.20518723,\n",
       "       10.49661458,  7.51226852,  9.85968045,  4.52890625,  5.3390625 ,\n",
       "        7.27615741, 28.00909624, 19.29175347, 16.95425347,  5.19296875,\n",
       "       18.18498264,  3.76302083, 15.67974537,  5.93376736, 23.10809179,\n",
       "       18.79001736, 16.04522569, 11.91377315,  8.22847222, 47.33837366,\n",
       "        6.48272569, 16.21308594,  7.86987847, 23.99774306,  7.490625  ,\n",
       "       20.38561198, 19.30579977, 16.76831597,  6.8485267 , 14.06816406,\n",
       "        4.97899306, 32.36935764, 13.63344907, 19.12237654, 25.62543403,\n",
       "        9.83402778, 30.34027778, 13.79479167, 13.74609375, 15.24097222,\n",
       "       10.26640625,  4.24704861, 27.54826389, 50.22157118, 33.30642361,\n",
       "       42.54980469, 30.0431713 , 20.71875   , 19.98090278,  6.83350622,\n",
       "        5.95147569,  8.54722222,  8.86006944,  6.89149306, 14.93402778,\n",
       "       37.1588748 , 25.10451389, 21.08472222, 18.52094907, 34.29366319,\n",
       "        3.89915365, 21.29722222, 13.334375  , 25.43064236])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:ComputerVision]",
   "language": "python",
   "name": "conda-env-ComputerVision-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}