retina.ipynb

{
 "cells": [
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 29,
   "metadata": {
    "collapsed": false
=======
   "execution_count": 21,
   "metadata": {
    "collapsed": true
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
<<<<<<< HEAD
    "import scipy.optimize as opt\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import pyretina\n",
    "\n",
    "from pyretina.optimize import multistart_until\n",
    "from pyretina.rogozhnikov_curve2 import RogozhnikovCurve2\n",
=======
    "import pandas as pd\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "import scipy.optimize as opt\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
    "\n",
    "%matplotlib inline\n",
    "\n",
    "DATA_PATH = \"./data/MC/\""
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 12,
=======
   "execution_count": 4,
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
<<<<<<< HEAD
    "from ipyparallel import Client\n",
    "\n",
    "rc = Client()\n",
    "\n",
    "lview = rc.load_balanced_view()"
=======
    "DATA_PATH = \"./data/MC/\""
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "\n",
    "events = list()\n",
    "r = re.compile(\"\\\\d+[_]\\\\d+\")\n",
    "\n",
    "event_names = list(set([ r.findall(item)[0] for item in os.listdir(DATA_PATH) if len(r.findall(item)) > 0 ]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%px\n",
    "\n",
    "import sys\n",
    "\n",
    "sys.path.append(\"./pyretina\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%px\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import pyretina\n",
    "from pyretina.optimize import multistart_until\n",
    "from pyretina.rogozhnikov_curve2 import RogozhnikovCurve2\n",
    "\n",
    "DATA_PATH = \"/home/mborisyak/notebook/pyretina/data/MC/\"\n",
    "    \n",
    "bounds = np.array([\n",
    "      [0, 2], # ax\n",
    "      [0, 2], # bx\n",
    "      [0, 2], # ay\n",
    "      [0, 2], # by\n",
    "      [-6000, 5000], # x0\n",
    "      [-6000, 5000], # y0\n",
    "      [5000, 6000],  # z0\n",
    "      [-0.1, 0.1] # gamma\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def fit_event(item):\n",
    "   \n",
    "\n",
    "        \n",
    "        results = list()\n",
    "        \n",
    "        for ei in range(tx.shape[0]):\n",
    "            f, f_jac, f_hess = rogozhnikov_curve2.RogozhnikovCurve2(\n",
    "                tx[ei, :], ty[ei, :], tz[ei, :], spline=True, spline_n=30).mse()\n",
    "\n",
    "            res = multistart_until(bounds, 1.0e+2, f, n_attempts = 250, f_jac=f_jac,\n",
    "                method=\"BFGS\", strict=False, solver_options={ \"maxiter\" : 100, \"gtol\" : 1.0e-5}\n",
    "            )\n",
    "\n",
    "            results.append(res)\n",
    "\n",
    "        return (item, results)\n",
    "    except Exception as e:\n",
    "        print e\n",
    "        return (item, [])"
=======
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def scan_dir(path):\n",
    "    import os\n",
    "    import re\n",
    "    \n",
    "    r = re.compile(r\"\\d+[_]\\d+\")\n",
    "    \n",
    "    events = list()\n",
    "    \n",
    "    for item in os.listdir(path):\n",
    "        try:\n",
    "            event_name = r.findall(item)[0]\n",
    "            events.append(event_name)\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "    return list(set(events))"
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fitted = lview.map(fit_event, event_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 2579\n",
      "Done: 195 elapsed: 589.342198 speed: 0.330893261938 events / sec\n",
      "left: 2.00131673381 hours\n",
      "Done: 212 elapsed: 649.478861 speed: 0.32642960786 events / sec\n",
      "left: 2.01421679948 hours\n",
      "Done: 233 elapsed: 709.604265 speed: 0.328357382078 events / sec\n",
      "left: 1.98462620984 hours\n",
      "Done: 246 elapsed: 769.732439 speed: 0.319599725612 events / sec\n",
      "left: 2.02770998728 hours\n",
      "Done: 267 elapsed: 829.891457 speed: 0.32173650916 events / sec\n",
      "left: 1.99611235883 hours\n",
      "Done: 291 elapsed: 890.052841 speed: 0.326957031903 events / sec\n",
      "left: 1.94385039482 hours\n",
      "Done: 307 elapsed: 950.21139 speed: 0.323095491733 events / sec\n",
      "left: 1.95332688712 hours\n",
      "Done: 324 elapsed: 1010.363263 speed: 0.320680500174 events / sec\n",
      "left: 1.95331143786 hours\n",
      "Done: 344 elapsed: 1070.529747 speed: 0.321348285635 events / sec\n",
      "left: 1.93196404364 hours\n",
      "Done: 361 elapsed: 1130.650543 speed: 0.319288514103 events / sec\n",
      "left: 1.9296375657 hours\n",
      "Done: 375 elapsed: 1190.782327 speed: 0.314926414596 events / sec\n",
      "left: 1.9440167412 hours\n",
      "Done: 394 elapsed: 1250.909148 speed: 0.31497391223 events / sec\n",
      "left: 1.92696734834 hours\n",
      "Done: 415 elapsed: 1311.040286 speed: 0.316545387603 events / sec\n",
      "left: 1.89897289505 hours\n",
      "Done: 430 elapsed: 1371.179873 speed: 0.313605455813 events / sec\n",
      "left: 1.90348870971 hours\n",
      "Done: 446 elapsed: 1431.34067 speed: 0.311598612808 events / sec\n",
      "left: 1.9014847167 hours\n",
      "Done: 464 elapsed: 1491.49627 speed: 0.3111019696 events / sec\n",
      "left: 1.88844834623 hours\n",
      "Done: 490 elapsed: 1551.619081 speed: 0.315801507791 events / sec\n",
      "left: 1.83747627374 hours\n",
      "Done: 511 elapsed: 1611.745313 speed: 0.317049878847 events / sec\n",
      "left: 1.81184249788 hours\n",
      "Done: 530 elapsed: 1671.907844 speed: 0.317009918806 events / sec\n",
      "left: 1.79542226568 hours\n",
      "Done: 544 elapsed: 1732.036163 speed: 0.314086305987 events / sec\n",
      "left: 1.79975302012 hours\n",
      "Done: 564 elapsed: 1792.215636 speed: 0.314699172324 events / sec\n",
      "left: 1.77859451644 hours\n",
      "Done: 581 elapsed: 1852.340294 speed: 0.313659215425 events / sec\n",
      "left: 1.76943629489 hours\n",
      "Done: 597 elapsed: 1912.469707 speed: 0.312165059343 events / sec\n",
      "left: 1.76366809506 hours\n",
      "Done: 615 elapsed: 1972.629214 speed: 0.311771054245 events / sec\n",
      "left: 1.74985954638 hours\n",
      "Done: 628 elapsed: 2032.727266 speed: 0.308946334183 events / sec\n",
      "left: 1.75417017288 hours\n",
      "Done: 646 elapsed: 2092.882304 speed: 0.308669292093 events / sec\n",
      "left: 1.73954603908 hours\n",
      "Done: 662 elapsed: 2153.024828 speed: 0.307476057487 events / sec\n",
      "left: 1.73184216147 hours\n",
      "Done: 679 elapsed: 2213.167514 speed: 0.306801685236 events / sec\n",
      "left: 1.7202571015 hours\n",
      "Done: 698 elapsed: 2273.306697 speed: 0.307043257633 events / sec\n",
      "left: 1.70171461842 hours\n",
      "Done: 720 elapsed: 2333.48187 speed: 0.308555457961 events / sec\n",
      "left: 1.67356912855 hours\n",
      "Done: 736 elapsed: 2393.602562 speed: 0.307487812614 events / sec\n",
      "left: 1.66492596924 hours\n",
      "Done: 757 elapsed: 2453.733192 speed: 0.308510979148 events / sec\n",
      "left: 1.64049627183 hours\n",
      "Done: 769 elapsed: 2513.868552 speed: 0.30590705797 events / sec\n",
      "left: 1.6435638364 hours\n",
      "Done: 790 elapsed: 2574.026011 speed: 0.306915477543 events / sec\n",
      "left: 1.61915732769 hours\n",
      "Done: 806 elapsed: 2634.200869 speed: 0.305979276841 events / sec\n",
      "left: 1.60958612977 hours\n",
      "Done: 826 elapsed: 2694.314488 speed: 0.306572823765 events / sec\n",
      "left: 1.58834836847 hours\n",
      "Done: 843 elapsed: 2754.451013 speed: 0.306053271138 events / sec\n",
      "left: 1.5756153183 hours\n",
      "Done: 867 elapsed: 2814.600531 speed: 0.308037854396 events / sec\n",
      "left: 1.54382180232 hours\n",
      "Done: 885 elapsed: 2874.752982 speed: 0.307853754496 events / sec\n",
      "left: 1.52850354652 hours\n",
      "Done: 900 elapsed: 2934.889324 speed: 0.306658384782 events / sec\n",
      "left: 1.52087440629 hours\n",
      "Done: 921 elapsed: 2995.020768 speed: 0.30751156671 events / sec\n",
      "left: 1.49768530818 hours\n",
      "Done: 937 elapsed: 3055.160219 speed: 0.30669537465 events / sec\n",
      "left: 1.48717962125 hours\n",
      "Done: 954 elapsed: 3115.301039 speed: 0.306231572916 events / sec\n",
      "left: 1.47401159387 hours\n",
      "Done: 970 elapsed: 3175.429478 speed: 0.305473139664 events / sec\n",
      "left: 1.4631219129 hours\n",
      "Done: 989 elapsed: 3235.562592 speed: 0.30566663198 events / sec\n",
      "left: 1.44492928065 hours\n",
      "Done: 1002 elapsed: 3295.70017 speed: 0.304033588495 events / sec\n",
      "left: 1.440813029 hours\n",
      "Done: 1025 elapsed: 3355.856364 speed: 0.305438738002 events / sec\n",
      "left: 1.41326758187 hours\n",
      "Done: 1041 elapsed: 3416.001752 speed: 0.304745420066 events / sec\n",
      "left: 1.40189874594 hours\n",
      "Done: 1057 elapsed: 3476.140585 speed: 0.304073879161 events / sec\n",
      "left: 1.39037847955 hours\n",
      "Done: 1073 elapsed: 3536.281937 speed: 0.303427052003 events / sec\n",
      "left: 1.37869491389 hours\n",
      "Done: 1085 elapsed: 3596.380724 speed: 0.301693175201 events / sec\n",
      "left: 1.37556973148 hours\n",
      "Done: 1100 elapsed: 3656.511298 speed: 0.300835484903 events / sec\n",
      "left: 1.36564120242 hours\n",
      "Done: 1113 elapsed: 3716.620802 speed: 0.299466514772 events / sec\n",
      "left: 1.35982556358 hours\n",
      "Done: 1134 elapsed: 3776.77261 speed: 0.300257281966 events / sec\n",
      "left: 1.33681650037 hours\n",
      "Done: 1151 elapsed: 3836.931947 speed: 0.299982158046 events / sec\n",
      "left: 1.32230086366 hours\n",
      "Done: 1169 elapsed: 3897.091554 speed: 0.299968861436 events / sec\n",
      "left: 1.30569108004 hours\n",
      "Done: 1190 elapsed: 3957.238655 speed: 0.300715767055 events / sec\n",
      "left: 1.28304989496 hours\n",
      "Done: 1209 elapsed: 4017.364067 speed: 0.300944468513 events / sec\n",
      "left: 1.26453746579 hours\n",
      "Done: 1223 elapsed: 4077.532648 speed: 0.299938755743 events / sec\n",
      "left: 1.25581192645 hours\n",
      "Done: 1246 elapsed: 4137.665595 speed: 0.301136798957 events / sec\n",
      "left: 1.22959989965 hours\n",
      "Done: 1265 elapsed: 4197.822019 speed: 0.301348534788 events / sec\n",
      "left: 1.21122208295 hours\n",
      "Done: 1281 elapsed: 4257.966189 speed: 0.300848624924 events / sec\n",
      "left: 1.19846170361 hours\n",
      "Done: 1293 elapsed: 4318.117759 speed: 0.299437907873 events / sec\n",
      "left: 1.1929759487 hours\n",
      "Done: 1316 elapsed: 4378.243047 speed: 0.300577987396 events / sec\n",
      "left: 1.16719569644 hours\n",
      "Done: 1334 elapsed: 4438.354936 speed: 0.300562649133 events / sec\n",
      "left: 1.15061979368 hours\n",
      "Done: 1356 elapsed: 4498.512163 speed: 0.301434799569 events / sec\n",
      "left: 1.12701726114 hours\n",
      "Done: 1372 elapsed: 4558.66125 speed: 0.300967326361 events / sec\n",
      "left: 1.11400058549 hours\n",
      "Done: 1389 elapsed: 4618.80187 speed: 0.300728069411 events / sec\n",
      "left: 1.09918424377 hours\n",
      "Done: 1406 elapsed: 4678.94878 speed: 0.300495541919 events / sec\n",
      "left: 1.08432002436 hours\n",
      "Done: 1425 elapsed: 4739.118442 speed: 0.300689549822 events / sec\n",
      "left: 1.06606816148 hours\n",
      "Done: 1442 elapsed: 4799.267362 speed: 0.300464203046 events / sec\n",
      "left: 1.05115128568 hours\n",
      "Done: 1460 elapsed: 4859.404616 speed: 0.300449517381 events / sec\n",
      "left: 1.03456093404 hours\n",
      "Done: 1477 elapsed: 4919.532698 speed: 0.300232444024 events / sec\n",
      "left: 1.01958038581 hours\n",
      "Done: 1492 elapsed: 4979.670355 speed: 0.299618889131 events / sec\n",
      "left: 1.00776171129 hours\n",
      "Done: 1515 elapsed: 5039.838854 speed: 0.300605518565 events / sec\n",
      "left: 0.983200697599 hours\n",
      "Done: 1531 elapsed: 5099.961976 speed: 0.300199440312 events / sec\n",
      "left: 0.96972569572 hours\n",
      "Done: 1553 elapsed: 5160.116173 speed: 0.300962861588 events / sec\n",
      "left: 0.946960693078 hours\n",
      "Done: 1568 elapsed: 5220.222179 speed: 0.300371060228 events / sec\n",
      "left: 0.934954696101 hours\n",
      "Done: 1587 elapsed: 5280.330077 speed: 0.300550092175 events / sec\n",
      "left: 0.916837368312 hours\n",
      "Done: 1605 elapsed: 5340.469997 speed: 0.300536798607 events / sec\n",
      "left: 0.900241024759 hours\n",
      "Done: 1628 elapsed: 5400.631899 speed: 0.301446887827 events / sec\n",
      "left: 0.876329056075 hours\n",
      "Done: 1642 elapsed: 5460.760869 speed: 0.300691968822 events / sec\n",
      "left: 0.865596041018 hours\n",
      "Done: 1660 elapsed: 5520.896994 speed: 0.300677281772 events / sec\n",
      "left: 0.849009197747 hours\n",
      "Done: 1679 elapsed: 5581.058659 speed: 0.300840433407 events / sec\n",
      "left: 0.831005317898 hours\n",
      "Done: 1701 elapsed: 5641.2266 speed: 0.301531596614 events / sec\n",
      "left: 0.808833606918 hours\n",
      "Done: 1720 elapsed: 5701.355647 speed: 0.301683216036 events / sec\n",
      "left: 0.790932668533 hours\n",
      "Done: 1737 elapsed: 5761.492663 speed: 0.301484948127 events / sec\n",
      "left: 0.775789605227 hours\n",
      "Done: 1751 elapsed: 5821.638663 speed: 0.300774968659 events / sec\n",
      "left: 0.764691294043 hours\n",
      "Done: 1772 elapsed: 5881.768835 speed: 0.301270456563 events / sec\n",
      "left: 0.744071188473 hours\n",
      "Done: 1796 elapsed: 5941.892013 speed: 0.302261180719 events / sec\n",
      "left: 0.719576359368 hours\n",
      "Done: 1814 elapsed: 6002.017367 speed: 0.302232319132 events / sec\n",
      "left: 0.703101510157 hours\n",
      "Done: 1836 elapsed: 6062.160208 speed: 0.302862869621 events / sec\n",
      "left: 0.681459860522 hours\n",
      "Done: 1860 elapsed: 6122.31728 speed: 0.303807264073 events / sec\n",
      "left: 0.657397784189 hours\n",
      "Done: 1879 elapsed: 6182.458487 speed: 0.303924935661 events / sec\n",
      "left: 0.639777858376 hours\n",
      "Done: 1898 elapsed: 6242.623011 speed: 0.304039374833 events / sec\n",
      "left: 0.62217818587 hours\n",
      "Done: 1920 elapsed: 6302.76655 speed: 0.304628644236 events / sec\n",
      "left: 0.600913797895 hours\n",
      "Done: 1935 elapsed: 6362.891613 speed: 0.304107907436 events / sec\n",
      "left: 0.58824149098 hours\n",
      "Done: 1950 elapsed: 6423.023001 speed: 0.303595880406 events / sec\n",
      "left: 0.575509199889 hours\n",
      "Done: 1971 elapsed: 6483.150366 speed: 0.304019375519 events / sec\n",
      "left: 0.555520149334 hours\n",
      "Done: 1990 elapsed: 6543.290779 speed: 0.304128802232 events / sec\n",
      "left: 0.537966512578 hours\n",
      "Done: 2010 elapsed: 6603.444828 speed: 0.304387067565 events / sec\n",
      "left: 0.51925844557 hours\n",
      "Done: 2029 elapsed: 6663.578085 speed: 0.304491559557 events / sec\n",
      "left: 0.501747168296 hours\n",
      "Done: 2047 elapsed: 6723.699239 speed: 0.304446003311 events / sec\n",
      "left: 0.485398974434 hours\n",
      "Done: 2064 elapsed: 6783.85167 speed: 0.304253139648 events / sec\n",
      "left: 0.470185963311 hours\n",
      "Done: 2079 elapsed: 6843.968017 speed: 0.303625663836 events / sec\n",
      "left: 0.458349485048 hours\n",
      "Done: 2099 elapsed: 6904.11616 speed: 0.304022766912 events / sec\n",
      "left: 0.438563646689 hours\n",
      "Done: 2115 elapsed: 6964.254288 speed: 0.303694151163 events / sec\n",
      "left: 0.424403592876 hours\n",
      "Done: 2135 elapsed: 7024.437216 speed: 0.303939455115 events / sec\n",
      "left: 0.40578257037 hours\n",
      "Done: 2149 elapsed: 7084.589164 speed: 0.30333559208 events / sec\n",
      "left: 0.393769961597 hours\n",
      "Done: 2161 elapsed: 7144.717796 speed: 0.302462348188 events / sec\n",
      "left: 0.383886165689 hours\n",
      "Done: 2178 elapsed: 7204.845836 speed: 0.302296977014 events / sec\n",
      "left: 0.368475034018 hours\n",
      "Done: 2195 elapsed: 7264.982888 speed: 0.302134673208 events / sec\n",
      "left: 0.353043447593 hours\n",
      "Done: 2220 elapsed: 7325.125994 speed: 0.303066898296 events / sec\n",
      "left: 0.329043596588 hours\n",
      "Done: 2238 elapsed: 7385.264618 speed: 0.303036286124 events / sec\n",
      "left: 0.312577161745 hours\n",
      "Done: 2258 elapsed: 7445.396348 speed: 0.30327507207 events / sec\n",
      "left: 0.294012514969 hours\n",
      "Done: 2279 elapsed: 7505.506432 speed: 0.303644148866 events / sec\n",
      "left: 0.274444061065 hours\n",
      "Done: 2302 elapsed: 7565.64656 speed: 0.304270511413 events / sec\n",
      "left: 0.252881700849 hours\n",
      "Done: 2326 elapsed: 7625.804137 speed: 0.305018071085 events / sec\n",
      "left: 0.230405292145 hours\n",
      "Done: 2348 elapsed: 7685.967049 speed: 0.305492856009 events / sec\n",
      "left: 0.210043100532 hours\n",
      "Done: 2367 elapsed: 7746.093263 speed: 0.305573809901 events / sec\n",
      "left: 0.1927157596 hours\n",
      "Done: 2391 elapsed: 7806.238647 speed: 0.306293882928 events / sec\n",
      "left: 0.170497111216 hours\n",
      "Done: 2408 elapsed: 7866.354885 speed: 0.306114229307 events / sec\n",
      "left: 0.155170833148 hours\n",
      "Done: 2427 elapsed: 7926.495399 speed: 0.306189146092 events / sec\n",
      "left: 0.137895881553 hours\n",
      "Done: 2445 elapsed: 7986.622028 speed: 0.306137330236 events / sec\n",
      "left: 0.121586682008 hours\n",
      "Done: 2462 elapsed: 8046.751074 speed: 0.305962394517 events / sec\n",
      "left: 0.10622220437 hours\n",
      "Done: 2480 elapsed: 8106.881784 speed: 0.305913321415 events / sec\n",
      "left: 0.0898947449325 hours\n",
      "Done: 2499 elapsed: 8167.009865 speed: 0.305987528848 events / sec\n",
      "left: 0.0726246010938 hours\n",
      "Done: 2519 elapsed: 8227.184859 speed: 0.306181041492 events / sec\n",
      "left: 0.0544340256517 hours\n",
      "Done: 2541 elapsed: 8287.321894 speed: 0.30661330868 events / sec\n",
      "left: 0.0344262798017 hours\n",
      "Done: 2557 elapsed: 8347.457017 speed: 0.30632121493 events / sec\n",
      "left: 0.0199500093799 hours\n",
      "Done: 2573 elapsed: 8407.559175 speed: 0.30603449327 events / sec\n",
      "left: 0.00544600920262 hours\n",
      "Done: 2578 elapsed: 8467.651747 speed: 0.304453128298 events / sec\n",
      "left: 0.000912382734678 hours\n"
     ]
    }
   ],
   "source": [
    "print \"total\", len(event_names)\n",
    "\n",
    "from time import sleep\n",
    "\n",
    "while not fitted.ready():\n",
    "    elapsed = fitted.elapsed\n",
    "    work_done = float(fitted.progress)\n",
    "    \n",
    "    speed = work_done / elapsed\n",
    "    left = len(event_names) - fitted.progress\n",
    "    eta = left / speed\n",
    "    \n",
    "    print \"Done:\", fitted.progress, \"elapsed:\", fitted.elapsed, \"speed:\", speed, \"events / sec\"\n",
    "    print \"left:\", eta / 3600, \"hours\"\n",
    "    import sys\n",
    "    sys.stdout.flush()\n",
    "    sleep(60.0)"
=======
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "event_names = scan_dir(DATA_PATH)"
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "results = [ r for r in fitted.get() if len(r) == 2 ]"
=======
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_event(item, data_path = DATA_PATH):\n",
    "    hits = pd.DataFrame.from_csv(\"%s/%s.hits.csv\" % (data_path, item))[['X', 'Y', 'Z']].values\n",
    "    tracks = pd.DataFrame.from_csv(\"%s/%s.tracks.csv\" % (data_path, item))\n",
    "    \n",
    "    tx = tracks[[ 'x%d' % i for i in range(11) ]].values\n",
    "    ty = tracks[[ 'y%d' % i for i in range(11) ]].values\n",
    "    tz = tracks[[ 'z%d' % i for i in range(11) ]].values\n",
    "    \n",
    "    pid = tracks['name'].values\n",
    "    \n",
    "    return hits, (tx, ty, tz, pid)"
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 134,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not (np.array())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
=======
   "execution_count": 14,
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
<<<<<<< HEAD
    "def clean(results):\n",
    "    for (item, result) in results:\n",
    "        params = np.zeros((len(result), 8))\n",
    "        errors = np.array([ opt.fun for opt in result ])\n",
    "        if not np.any(np.isnan(errors)) and np.all(errors < 100.0):\n",
    "            for i in xrange(params.shape[0]):\n",
    "                params[i, :] = result[i].x\n",
    "            yield (item, params)"
=======
    "def load_events(data_path = DATA_PATH):\n",
    "    events = list()\n",
    "    \n",
    "    for item in scan_dir(data_path):\n",
    "        try:\n",
    "            events.append(load_event(item, data_path))\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    return events"
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cleaned = list(clean(results))"
=======
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "events = load_events()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2573"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(events)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "zs = np.hstack([ hits[:, 2] for hits, _ in events ])"
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for item, params in cleaned:\n",
    "    np.savetxt(\"%s.fitted.csv\" % item, params, delimiter = \",\")"
=======
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2648.4\n"
     ]
    }
   ],
   "source": [
    "before_cut = np.max(zs[zs < 5000]) + 0.1\n",
    "print before_cut"
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": null,
=======
   "execution_count": 36,
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
<<<<<<< HEAD
    "def max_error(results):\n",
    "    for (item, result) in results:\n",
    "        params = np.zeros((len(result), 8))\n",
    "        errors = np.array([ opt.fun for opt in result ])\n",
    "        if not np.any(np.isnan(errors)) and np.all(errors < 100.0):\n",
    "            for i in xrange(params.shape[0]):\n",
    "                params[i, :] = result[i].x\n",
    "            yield (item, params)"
=======
    "def cut(events):\n",
    "    from sklearn.linear_model import LinearRegression\n",
    "    \n",
    "    lr = LinearRegression(fit_intercept=False)\n",
    "    \n",
    "    for hits, (tx, ty, tz, pid) in events:\n",
    "        hits_ = hits[hits[:, 2] < before_cut, :]\n",
    "        \n",
    "        params = np.zeros((tx.shape[0], 2))\n",
    "        \n",
    "        for t in xrange(tx.shape[0]):\n",
    "            cut = tz[t, :] < before_cut\n",
    "            \n",
    "            tx_ = tx[t, :][cut]\n",
    "            ty_ = ty[t, :][cut]\n",
    "            tz_ = tz[t, :][cut].reshape(-1, 1)\n",
    "        \n",
    "            assert tz_.shape[0] > 1\n",
    "        \n",
    "            lr.fit(tz_, tx_)\n",
    "            cx = lr.coef_\n",
    "            \n",
    "            print tz_, tx_\n",
    "            \n",
    "            assert lr.score(tz_, tx_) >= 0.98, \"Low score %f\" % lr.score(tz_, tx_)\n",
    "        \n",
    "            lr.fit(tz_, ty_)\n",
    "            cy = lr.coef_\n",
    "        \n",
    "            assert lr.score(tz_, ty_) >= 0.98\n",
    "        \n",
    "            params[t, :] = cx, cy\n",
    "\n",
    "        yield hits_, params"
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 151,
=======
   "execution_count": 37,
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
<<<<<<< HEAD
     "data": {
      "text/plain": [
       "790"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cleaned)"
   ]
=======
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.74088  51.255  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.281882  62.9832  ]\n",
      "[[  -11.9815]\n",
      " [ 1889.22  ]] [  0.520724  91.3356  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.446999  78.8592  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.430757  60.7484  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.472913  75.8367  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.357196 -23.2837  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.338909 -12.8681  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.41269  89.752  ]\n",
      "[[   19.9478]\n",
      " [ 1417.95  ]] [   0.358979 -104.137   ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [   0.359045  182.111   ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [   0.530419  141.006   ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.315245  45.5474  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.579144 -17.4054  ]\n",
      "[[   19.9478]\n",
      " [ 1417.95  ]] [   0.406143 -109.05    ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.500624 -36.094   ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.456859  43.2812  ]\n",
      "[[  -11.9815]\n",
      " [ 1889.22  ]] [   0.186224  169.645   ]\n",
      "[[   19.9478]\n",
      " [ 1417.95  ]] [  0.239901 -28.5238  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.308187 -15.9393  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.461332  36.925   ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.706924  12.3712  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [   0.447681  203.714   ]\n",
      "[[   19.9478]\n",
      " [ 1417.95  ]] [  0.966518  66.4455  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.512203 -75.0744  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.486924  90.5923  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [   0.446557  130.207   ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.277544  99.4238  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.551558 -14.9763  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.420707  31.431   ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.411643  31.0901  ]\n",
      "[[  -11.9815]\n",
      " [ 1389.22  ]] [  0.260105 -17.9162  ]\n",
      "[[   0]\n",
      " [1400]] [-29.636  -88.7591]\n"
     ]
    },
    {
     "ename": "AssertionError",
     "evalue": "Low score 0.497478",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-37-21378c3f650c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mcutted\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcut\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevents\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-36-36aa4b93d6f5>\u001b[0m in \u001b[0;36mcut\u001b[1;34m(events)\u001b[0m\n\u001b[0;32m     23\u001b[0m             \u001b[1;32mprint\u001b[0m \u001b[0mtz_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtx_\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m             \u001b[1;32massert\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtz_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtx_\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>=\u001b[0m \u001b[1;36m0.98\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"Low score %f\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtz_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtx_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     26\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m             \u001b[0mlr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtz_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mty_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAssertionError\u001b[0m: Low score 0.497478"
     ]
    }
   ],
   "source": [
    "cutted = list(cut(events))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
>>>>>>> 8d12dcce2e1cac04f3d4a24c904e979ca5f452ea
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}