Reinforcement_learning_MsPacman.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import threading\n",
    "from queue import Queue\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env=gym.make('MsPacman-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs=env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(210, 160, 3)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "img=env.render(mode='rgb_array')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fc46fa1a0d0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAM4AAAD8CAYAAAA/rZtiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAASMklEQVR4nO3dbcwcV3nG8f+FIYiGVEl4sWgcmgQF1BAlxrgQQWOlpQFjtZi0AsUfaApRAxJBoFIJB6Q2AiGllECN1KZ1FKuhggTaQIkqk2JFlKRqE5IYY2yMiRMMeWLLLoTyroCdux9m1h6vd/3snpnZPTO+ftJqd8/O7pyZ3XvPmbNn7lVEYGbTedq8K2DWRQ4cswQOHLMEDhyzBA4cswQOHLMErQWOpNWSdkvaI2l9W+sxmwe18TuOpCXAt4HLgQXgAWBdRHyz8ZWZzUFbLc4rgD0R8WhE/BK4HVjb0rrMZu7pLb3uWcBjlfsLwCvHLSzJ0xcsR9+PiOeNeqCtwNGIsmOCQ9I1wDUAZ512Gve/7W0tVcUszbING7477rG2AmcBOLtaB2BfdYGI2AhsBLh46dJjgmrZHS9oqVrpFv54/3FlOdYzR8P7Ltf9Nuo9HqetY5wHgPMlnSvpFOBK4M6W1mU2c620OBFxSNK1wH8AS4BNEbGzjXWZzUNbXTUiYjOwua3XN5snzxwwS9Bai9OkSQ4uF1um7uNN1HPax5uo5yzWmeO+S/mMTMMtjlmCVqbcTOvipUtj87p1R+7nOFzp4eh0XR2OXrZhw0MRsXLUsm5xzBI4cMwSOHDMEjhwzBI4cMwSdOJ3nElMM0EP5jOyM20dc9WFfeffccwy5MAxS+DAMUvgwDFL0JvBgbpmMVGxr07GfecWxyyBW5xSE99wuX9LtqUr+67JdSS3OJLOlvRlSbsk7ZT07rL8ekmPS9pWXtY0VluzTNRpcQ4B742IrZJOAx6StKV87OMR8dH61TPLU3LgRMR+YH95+yeSdlEkIjTrvUYGBySdA7wMuL8sulbSdkmbJJ3RxDrMclJ7cEDSs4E7gPdExI8l3QR8iCJz54eAG4Hj0nQOZ/Ksq+2Dy64cAOeoj/uuVosj6RkUQfOpiPgcQEQciIjDEfEUcDNFAvbjRMTGiFgZESuf86xn1amG2czVGVUTcAuwKyI+VimvfjVcAexIr55Znup01V4NvAX4hqRtZdn7gXWSllN01fYCb69VQ7MM1RlV+y9G/yuBs3da73Vi5kAOCQlnkbivrwkJc3j/Jl1mUp6rZpbACQnNSk5IaNYyB45ZAgeOWQIHjlmCLIejF8uhNY9Tc1Nyos1iHW1ou97zOrW6yf3rFscsgQPHLIEDxyyBA8csgQPHLEGWo2opmh41a2PaT9eS7g10Yd/Met+6xTFL0JsWp+43TNcS4s1SF/bNrPetWxyzBE1kudkL/AQ4DByKiJWSzgQ+A5xDcfr0myPih3XXZZaLplqc342I5ZVzF9YDd0fE+cDd5X2z3mirq7YWuLW8fSvwxpbWYzYXTQwOBPAlSQH8Y0RsBJaWKXKJiP2Snn+iF9j+w2dkf+DchQPkeelKvZusZxOB8+qI2FcGxxZJ35rkSdVMniw5vYFqnNzuu/A/j7l/yY7L5lKPk0XtrlpE7CuvDwKfp8jceWCQmLC8PjjieUcyefK0U+tW46Q2HDTjyqw5dVPgnlr+xQeSTgVeS5G5807gqnKxq4Av1FmPjTcIkEt2XHaklRncdvC0p25XbSnw+SIbLk8HPh0Rd0l6APispKuB7wFvqrkem0A1UBw07aoVOBHxKHDxiPIfAK+p89pVs0g2l2PivkmWWdh99Ha1lRncznG75vH+NbGOqt5MuTnZubWZrSwSEuqUZcHSd7W6jq7OTF7MYkHSl9G1ubx/C+udkNCsSQ4cswQOnB6oDkOPurbmOXA6zkEzH1mMql10xq/YPEWyuCYODGeR/M8JCfMydeLEDeMfc4tjlsCBY5bAgWOWwIFjlsCBY5Ygi1G1JnRhSk1KHdvejklGmvq6b+twi2OWoDctTo7fgsO6UMdRulBvJyQ06wAHjlmC5K6apJdQZOscOA/4S+B04M+A/y3L3x8Rm5NraJah5MCJiN3AcgBJS4DHKbLcvBX4eER8tJEammWoqcGB1wCPRMR3y8QdU1ksIWETkwi7cIDbVV3dt3Xq3dQxzpXAbZX710raLmmTpDMaWodZNmoHjqRTgDcA/1IW3QS8iKIbtx+4cczzrpH0oKQHeepndathNlNNtDivB7ZGxAGAiDgQEYcj4ingZorMnsdxJk/rsiYCZx2Vbtog9W3pCorMnma9UmtwQNKvAZcDb68Uf0TScop/Mdg79Fhr2k6aN4vEfbnKISFhbvu2bibPnwPPGSp7S60amXVAJxISdvWbelqTtGrj/PelLz3m/qvu3dn6Ortm6s+RExL223DQjCuz5jhwOm4QIK+6d+eRVmZw28HTHgdOj1QDxUHTLgdOj1SPayY9xrE0WZzINm1CwhTzSJo3y4PsebY2Xd23i9XbCQnNGubAMUvgwDFL4MDpgeow9Khra14WgwNN6MLsgjbqOIugOVn37Ym4xTFL4MAxS9CbrlqO3YdhKXXM4U+b+rpv63CLY5bAgWOWwIFjlmCiwCnTPB2UtKNSdqakLZIeLq/PKMsl6ROS9pQpola0VXmzeZnoDFBJq4CfAp+MiAvLso8AT0TEDZLWA2dExPskrQHeBawBXglsiIhXnvD1FzkDtAld+C3CxpvL+1f3DNCIuAd4Yqh4LXBreftW4I2V8k9G4T7g9KHMN9aC7Tfcc+QyuG/tqXOMszQi9gOU188vy88CHqsst1CWHcMJCZszHCTbb7iHi9avcvC0qI3BgVHJo4/rDzohYTsuWr8KOBo81o46gXNg0AUrrw+W5QvA2ZXllgH7aqzHFnHR+lXHBIy1r87MgTuBq4AbyusvVMqvlXQ7xeDAjwZdulRNJLSru45ZJCSsu45RwZPjds3j/WtiHVUTBY6k24DLgOdKWgD+iiJgPivpauB7wJvKxTdTjKjtAX5O8X85Zr3SiYSETTgZhqOHu2l9OsbJbTi6N5M8T2Y+rpk9T7npqT61Njly4PTAcJA4aNrnrlpPOFhmK4vAmTYh4TwO7JvI6t/EAW7dE9uaWOc8trsJ0+47JyQ0a5gDxyyBA8csgQPHLEEWgwNNyOEAtuk6pNZj1uvMcd85IaFZhnrT4tT9hmniGyqHOsxjnTm8hvOqmXWAA8csgQPHLIEDxyyBA8cswaKjapI2AX8AHKwkI/wb4A+BXwKPAG+NiP+TdA6wC9hdPv2+iHjHtJVqY4SkL2d89mU7pjWLkbtpJoFO0uL8E7B6qGwLcGFEXAR8G7iu8tgjEbG8vEwdNGZdsGjgjMriGRFfiohD5d37KFJAmZ00mjjGeRvwxcr9cyV9TdJXJF067knVTJ4/+MUvGqiG2ezUmjkg6QPAIeBTZdF+4IUR8QNJLwf+TdJLI+LHw8+NiI3ARoCLly6df6odsykkB46kqygGDV4TZY6piHgSeLK8/ZCkR4AXAw/WqWQTyeaaSJrXRD2nef6o15h2HU0kJFzMLPZdDkkPq5K6apJWA+8D3hARP6+UP0/SkvL2ecD5wKPJtTPL1CTD0aOyeF4HPBPYIgmODjuvAj4o6RBwGHhHRAz/PcjUJvlmWGyZuo9PYhYTFaddRxe2e5LXaGI7mhzKXzRwImLdiOJbxix7B3BH3UqZ5c4zB8wSOHDMEjhwzBL05gzQqZPNdfRsy3kkJMxhflxu769bHLMEDhyzBA4cswQOHLMEnRgcmEWyuRznW6XWYxqz+PPclHrk+h4PuMUxS9CJFmcWw6FdmW/VtFnMA2yqHjmtwy2OWQIHjlkCB45ZAgeOWQIHjlmCToyq5WoeEy7noe525mKmv+NI2iTpoKQdlbLrJT0uaVt5WVN57DpJeyTtlvS65JqZZSw1kyfAxysZOzcDSLoAuBJ4afmcvx8k7zDrk6RMniewFrg9Ip6MiO8Ae4BX1KifWZbqDA5cK2l72ZU7oyw7C3isssxCWXYcZ/K0LksdHLgJ+BAQ5fWNFKlwNWLZkVk6c8vkOet/LZ6kDrOoxzzWOUk9ch84SWpxIuJARByOiKeAmznaHVsAzq4sugzYV6+KZvlJanEkvSAiBl8RVwCDEbc7gU9L+hjwGxSZPL9au5YzkMM3XK6TPGeha5M8UzN5XiZpOUU3bC/wdoCI2Cnps8A3KZKxvzMiDjdWW7NMNJrJs1z+w8CH61TKLHeecmOWwIFjlqA3c9XaPrg8mQ6ic1hnjnWocotjlsCBY5bAgWOWwIFjlqATgwM5/HnuLBL35frnuTn8sW0v/jzX7GSn8p/W5+ripUtj87qjExRyG3q0k8NxLdKGDQ9FxMpRy7rFMUvgwDFL4MAxS+DAMUuQ5XB0X/J4WX+5xTFLkJqQ8DOVZIR7JW0ry8+R9IvKY//QZuXN5mXR33EkrQJ+CnwyIi4c8fiNwI8i4oOSzgH+fdRyi6xj/j8mmR1v7O84k5w6fU8ZEMeRJODNwO/VqV2KLVt+G4DLL3/gyO3B/Wleo87zrR13rVgBwOqtW+dck/HqHuNcChyIiIcrZedK+pqkr0i6tObrjzT4wA9/6AePTfMaqc+3dty1YgWrt25l9dat3LVixZEgyk3dwFkH3Fa5vx94YUS8DPhzilRRvz7qidVMntOudPCBr7Y6qa+R+nxrx3ArMwig3CQPR0t6OvBHwMsHZRHxJPBkefshSY8ALwaOC45qJs+6xzh1A8ABlLdB8OTUdavzO87vA9+KiIVBgaTnAU9ExGFJ51EkJHy0Zh0XVfcD74DJTzVQcmxxJhmOvg34H+AlkhYkXV0+dCXHdtMAVgHbJX0d+FfgHREx6T8dmAHjgyanAEpNSEhE/OmIsjuAO+pXazruqvVPbl2zYb2aOVAdMJjH862+4RamGjw5BVJnA2cwHF33NSx/g+HpnHQ2cAaGP/zTBkPd51uzcm1hhmVx6rSn3FimfOq0WZMcOGYJHDhmCbI8A9Tm796/PTo/99L33DvHmuTJLY4dZxA0g4CpBpEVHDh2jOGgcfCM5sAxS+DAMUvgwLFjDHfNhrtuVvDMARvJo2rACWYOOHDMxvOUG7MmOXDMEkxy6vTZkr4saZeknZLeXZafKWmLpIfL6zPKckn6hKQ9krZLyud8V7OGTNLiHALeGxG/BVwCvFPSBcB64O6IOB+4u7wP8HqKJB3nA9cANzVea7M5WzRwImJ/RGwtb/8E2AWcBawFbi0XuxV4Y3l7LUW63IiI+4DTJfm/Ca1XpjrGKVPhvgy4H1gaEfuhCC7g+eViZwGPVZ62UJaZ9cbEs6MlPZsig817IuLHRdro0YuOKDtuuFnSNRRdObPOmajFkfQMiqD5VER8riw+MOiCldcHy/IF4OzK05cB+4ZfMyI2RsTKcePkZjmbZFRNwC3Aroj4WOWhO4GryttXAV+olP9JObp2CcVfgPgv1qxfIuKEF+B3KLpa24Ft5WUN8ByK0bSHy+szy+UF/B3wCPANYOUE6whffMnw8uC4z6yn3JiN5yk3Zk1y4JglcOCYJXDgmCVw4JglyCWv2veBn5XXffFc+rM9fdoWmHx7fnPcA1kMRwNIerBPswj6tD192hZoZnvcVTNL4MAxS5BT4GycdwUa1qft6dO2QAPbk80xjlmX5NTimHXG3ANH0mpJu8vkHusXf0Z+JO2V9A1J2yQ9WJaNTGaSI0mbJB2UtKNS1tlkLGO253pJj5fv0TZJayqPXVduz25Jr5toJYtN+W/zAiyhOP3gPOAU4OvABfOsU+J27AWeO1T2EWB9eXs98NfzrucJ6r8KWAHsWKz+FKeUfJHi9JFLgPvnXf8Jt+d64C9GLHtB+bl7JnBu+Xlcstg65t3ivALYExGPRsQvgdspkn30wbhkJtmJiHuAJ4aKO5uMZcz2jLMWuD0inoyI7wB7KD6XJzTvwOlLYo8AviTpoTKXAoxPZtIVfUzGcm3ZvdxU6Tonbc+8A2eixB4d8OqIWEGRU+6dklbNu0It6up7dhPwImA5sB+4sSxP2p55B85EiT1yFxH7yuuDwOcpmvpxyUy6olYyltxExIGIOBwRTwE3c7Q7lrQ98w6cB4DzJZ0r6RTgSopkH50h6VRJpw1uA68FdjA+mUlX9CoZy9Bx2BUU7xEU23OlpGdKOpciA+1XF33BDEZA1gDfphjN+MC865NQ//MoRmW+DuwcbANjkpnkeAFuo+i+/IriG/jqcfUnIRlLJtvzz2V9t5fB8oLK8h8ot2c38PpJ1uGZA2YJ5t1VM+skB45ZAgeOWQIHjlkCB45ZAgeOWQIHjlkCB45Zgv8HMgj+t1lPTZsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(210, 160, 3)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Discrete(9)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_enviroment(env,figsize=(5,4)):\n",
    "    plt.close()\n",
    "    plt.figure(figsize=figsize)\n",
    "    img=env.render(mode='rgb_array')\n",
    "    plt.imshow(img)\n",
    "    plt.axis('off')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Discrete(9)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()\n",
    "for step in range(110):\n",
    "    env.step(3)\n",
    "for step in range(40):\n",
    "    env.step(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAALQAAADnCAYAAAC313xrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAHwUlEQVR4nO3dvWrlRhjG8VHITSy7KdO6MKRZsAuDwX3AZW5h3cWXsOnONaQ0bL+w4GINbgwu9hayS64iKIUZo6Mj6Uiar3ce/X8QfD6tOfKzb2ak0ZymbVsHqPipdAOAmAg0pBBoSCHQkEKgIeXnqSebpuEQCMxp27YZe24y0N8/fIjfGiChyUD3vfv0JlU7Vvv++78Hj1lsp0X9fWd1vw39jccsCvQxF9fvDx67v3uMuQlgUrRB4VCYpx4HUogSaB/afjX29wk1colWocfCDOQU1If2nfU/3GEV7t/3r+0PPI4NTGIMXEK3MWfgubSdObZpcd8NtSHm4JQTK5DSTE0f/XFzs/fk2L+cbh+6W5m791N1QThst16th+3e7nbrTqws4cPbDa6/z6AQuUQLtA/tsb40kFKUPrSvyt3qPPQYkFrULkf359hjQEoc5YCUqHM5hiyZWOJcmZH20jZaVcO+S93GKCdWPKuHfbAddDkghUBDCoGGlOSDwlA5Jtio2uK+o0JDivkKHaMiWK8qqWxx3wUFurYPC310OSCFQEMKgYaU5IPC1P3sLQ58YlHcd1RoSCHQkEKgIYVAQ0ry+dCpFyrJsSCL6kIzFv5+c18zFxUaUqIsNAOktGShGSo0pBBoSCHQkEKgIWXRYbtjazCUuMRnzZoaObaRQup2l7pEK+b+pUJDCoGGFAINKQQaUgg0pGRffTR0pJ7i9Htti6l4Neyb3PuWCg0p5q8pzFEta6nIfTXsm9z7lgoNKQQaUgg0pBBoSJkcFNYwWKphYFRKLe1e2s52N/4cFRpSCDSkEGhIIdCQUnyhmdBtsNBMuufnvmYKC80AASYXmml++Wv8yUhqnemGFyX+fu0/f7LQDLaBQEMKgYYUAg0pk4ftcixUErrNNVhoxhYWmgFGEGhIIdCQQqAhhUBDivmFZnJY08bUn2POyF9134agQkOK+YVmcqihjUNqaDcLzQABCDSkJO9yYNjF9fu9+/d3j4VaooUKXUA/zGOPYbmghWZiTCqpYWATkw/u/d3jwe2L6/dRK3Wt+/ZYu1loxphuaOlqxEUfuoBu94KuRlxUaEgh0IXQ7Uij+FwOC4vA5J5v4AeBPsj+9tLuh4WFZqztW/rQBfjg0peOL2ihmRpme8Uw5/8CS3Qr8tDPFNu0bGmOWGjGGB/asZ9Yj0BDCoGGFAINKVEXmlmjxGIoqoOrvlr3bUi7qdCQQqAhhUBDCmcKK8ZVL4eKz+WwoIY29o1d9WIt1KzLAQQg0BXy1fnv/z69PuZvb32SEwvNuHVtLL2Y+P3do3vn3riL6/3b1uT++zMorBRTT4fR5YAUAl0xLuM6RKAr1b0goH9xwJbx1cgVOtZnzhlsvhoZwbjiZRyBrhShHkagIYVAQ0rQiZUYC5VslYUFdHL8/XJnhAoNKRy2QxAO2wEJEWhIIdCQQqAhJepCMyUGdDFW6YwxsAmd8B9jmyU+dwwxL5agQkMKgYYUAg0pBBpSNrHQTOzB05rfsZTVwa71gScVGlI2sS5HaBtKfIYY27TwO3LvOyo0pBBoSCHQkEKgIYVAQ8qioxwpRqwWjoLEoPI5lspxJGXJ5CUqNKQQaEgh0JBCoCFlEyv455ictHQbMRaaOSbH5CQLi9l0UaEhZRMVOscEm6XbOPZ6CxOL5vyOGJ8j5iFPKjSkEGhIIdCQQqAhJfs1hcfUenVIiYVmLMwfsfb3pUJDCoGGFAINKQQaUsyfKbQ4H2FtO5bI8aVBa9qR4nMzlwMYYb5C1zIfIbYYcyCszAfJuQ0qNKQQaEgh0JBCoCGFQEOK+aMca5SYKFRCzG+PUkGFhhQCDSkEGlIINKSYHxRa/BatHO0osc057bA+YKZCQ4r5Cm2hIlidnJSDlXbMFVShv338+vqfvw+UtDrQ/fB++/jVndye7wUcyG1VoLuBPbk933v85PZ87zEgp2iDQkIMC6INCse6GakHFVYGLVsdOFpoQxeH7SAlKNC+m0F3A1ZQoSFlVaDHKjKVGqWtHhSe3J7vn+d/+nXwdam/dCbHgixWvzTIwhf68KVBQEJN27ajT/64udl70tohGmxDv4K/3e2asddSoSGFQEMKgYYUAg0piw7bsQ4ErKNCQwqBhpTJ49BN04w/CRTStu3ocejoF8l++fKbc865y8un19v+/pLfEfJ+pPH59NQ559zV83PhloyLWqF9EP3tvjmhHArzkvcjjc+np69BLh3sqQodtQ/tg9it0mt/x9r3I41+eK+en1+DbUnSdTlCg0mwbfOhttQFSRro0CASZHuGuh6WcNgOs42F2VKwkwa6258u8X7EZym8Q7JUaIJdv35F7vabLfWhowf68vKJvvNGXD0/mwqzcwkrdD+US0Ma+n7EZbUi93HqG9XJdmIFKI1AQwqBhhTzX0mh5mF35pxz7uzm4eCxMWOv7T6OFwQ6o7HgDgVz6LXdfwwPuzP3sDsj1D10OTKaG75ucP17+pW9/zheEGhjhrokmI9AQwqBNmSqOve7GFTyYZwpzGTOgHBOSDnKMX2mkECjOpz6xmYQaEgh0JBCoCGFQEMKgYYUAg0pBBpSCDSkEGhIIdCQQqAhhUBDCoGGFAINKQQaUiYn+AO1oUJDCoGGFAINKQQaUgg0pBBoSPkfkUhGL49i7MMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_enviroment(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs,reward,done,info=env.step(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(210, 160, 3)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ale.lives': 3}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "env=gym.make('CartPole-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs=env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4,)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs,reward,done,info=env.step(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4,)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.00289301, -0.21844276,  0.03194412,  0.29267587])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAADICAYAAACuyvefAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAADQklEQVR4nO3dwU0CQQBAUdfQhHVgGdYBNWEdlqF1WMZ6MUrUBQVxP/hewmGZkMxh8jPDLmEYx/EKoOJ67gkAbBMlIEWUgBRRAlJECUgRJSBlsWfc8wLAKQxTA3ZKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkLOaeAJfh6X796b3lajPDTDh3dkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKHESnubmUKIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKHO2rP6KEQ4kSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKQIoo8euWq83cU+CMiRKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKThmH41uuQz8GUxdwT4Lw9blZXD8+rt+u7m/sZZ8MlsFPiKNtB+uoafkqUgBRR4igfj2uObxxrGMdx1/jOQS7bT76Qfty8H9tu1/vDtGfdcfkmF5coMemUd8lE6d+bXFyOb0CKKAEpogSkiBKQIkpAiigBKX77xiS37ZmDnRKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKaIEpIgSkCJKQIooASmiBKSIEpAiSkCKKAEpogSkiBKQIkpAiigBKYs948OfzALglZ0SkCJKQIooASmiBKSIEpAiSkDKC5WZK8Nx0A4GAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_enviroment(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image,ImageDraw\n",
    "\n",
    "try:\n",
    "    from pyglet.gl import gl_info\n",
    "    openai_cart_pole_rendering=True\n",
    "except Exception:\n",
    "    openai_cart_pole_rendering=False\n",
    "    \n",
    "def render_cart_pole(env,obs):\n",
    "    if openai_cart_pole_rendering:\n",
    "        return env.render(mode='rgb_array')\n",
    "    else:\n",
    "        img_w = 600\n",
    "        img_h = 400\n",
    "        cart_w = img_w // 12\n",
    "        cart_h = img_h // 15\n",
    "        pole_len = img_h // 3.5\n",
    "        pole_w = img_w // 80 + 1\n",
    "        x_width = 2\n",
    "        max_ang = 0.2\n",
    "        bg_col = (255, 255, 255)\n",
    "        cart_col = 0x000000 # Blue Green Red\n",
    "        pole_col = 0x669acc # Blue Green Red\n",
    "\n",
    "        pos, vel, ang, ang_vel = obs\n",
    "        img = Image.new('RGB', (img_w, img_h), bg_col)\n",
    "        draw = ImageDraw.Draw(img)\n",
    "        cart_x = pos * img_w // x_width + img_w // x_width\n",
    "        cart_y = img_h * 95 // 100\n",
    "        top_pole_x = cart_x + pole_len * np.sin(ang)\n",
    "        top_pole_y = cart_y - cart_h // 2 - pole_len * np.cos(ang)\n",
    "        draw.line((0, cart_y, img_w, cart_y), fill=0)\n",
    "        draw.rectangle((cart_x - cart_w // 2, cart_y - cart_h // 2, cart_x + cart_w // 2, cart_y + cart_h // 2), fill=cart_col) # draw cart\n",
    "        draw.line((cart_x, cart_y - cart_h // 2, top_pole_x, top_pole_y), fill=pole_col, width=pole_w) # draw pole\n",
    "        return np.array(img)\n",
    "def plot_cart_pole(env,obs):\n",
    "    plt.close()\n",
    "    img=render_cart_pole(env,obs)\n",
    "    plt.imshow(img)\n",
    "    plt.axis('off')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAADtElEQVR4nO3c0WnCUBiAUVNconPYMZxDZ2rm6BrO0THSt1LQqtSPJJVzIGByQf6H8HEfkgzTNG0AeNzL0gMAPAtBBYgIKkBEUAEiggoQ2d5Y9wgAwLnh0kU7VICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiAgqQERQASKCChARVICIoAJEBBUgIqgAEUEFiGyXHgDucRqPZ9d2h/cFJoHf2aECRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBZfUuvccPaySoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiKACRAQVICKo/Eu7w/vSI8AZQQWICCpARFABIoIKEBFUgIigAkQEFSAiqAARQQWICCpARFABIoIKEBFUgIigAkQEFSAiqAARQWXVTuNx6RHgboIKEBFUgIigAkQEFSAiqAARQQWICCpARFABIoIKEBFUgIigAkQEFSAiqAARQQWICCqzG4bh7uPR/4A5CSpAZLv0AHDLx+fh+/f+ddy8HccFp4Hf2aGyaj9jeukc1kRQASKCChARVFZt/zpePYc1GaZpurZ+dRH+Ys7HmW7c3/BXF29iO1SAiKACRAQVICKoABFBBYgIKkBEUAEiggoQEVSAiM/3MTtvL/Gs7FABIoIKEBFUgIigAkQEFSAiqAARQQWICCpARFABIoIKEBFUgIigAkQEFSAiqAARQQWICCpARFABIoIKEBFUgIigAkQEFSAiqAARQQWICCpARFABIoIKEBFUgIigAkQEFSAiqAARQQWICCpARFABIoIKEBFUgIigAkQEFSAiqAARQQWICCpARFABIoIKEBFUgIigAkQEFSCyvbE+zDIFwBOwQwWICCpARFABIoIKEBFUgIigAkS+ADINIhFmPazhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_cart_pole(env,obs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs=env.reset()\n",
    "while True:\n",
    "    obs,reward,done,info=env.step(0)\n",
    "    if done:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fc46006ac90>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAARy0lEQVR4nO3df6xcZ53f8fenTgh0QZuE3ESu7dTZXVclWxWH3gav0j+ygd0N0apmJaiSVouFIjmVggQSaptspW6QGmlX6pIVahvhVVJMRQnpAooVpWVdE7TiDxJuwBgbk40Bi9y1FV9KEkCoaRO+/WOeC7PO2Hd8f2Tuc+f9ko7OOc95zsz3USYfHz/njCdVhSSpH39r0gVIki6MwS1JnTG4JakzBrckdcbglqTOGNyS1Jk1C+4kNyd5OsmJJHet1ftI0rTJWjzHnWQT8FfAbwHzwFeB26rqW6v+ZpI0Zdbqivt64ERVfbeq/i/wELB7jd5LkqbKRWv0uluAZ4f254G3n6vzFVdcUdu3b1+jUiSpPydPnuQHP/hBRh1bq+Ae9WZ/Y04myV5gL8DVV1/N3NzcGpUiSf2ZnZ0957G1miqZB7YN7W8FTg13qKp9VTVbVbMzMzNrVIYkbTxrFdxfBXYkuSbJ64BbgQNr9F6SNFXWZKqkql5O8gHgC8Am4MGqOrYW7yVJ02at5ripqseAx9bq9SVpWvnNSUnqjMEtSZ0xuCWpMwa3JHXG4JakzhjcktQZg1uSOmNwS1JnDG5J6ozBLUmdMbglqTMGtyR1xuCWpM4Y3JLUGYNbkjpjcEtSZwxuSeqMwS1JnVnRT5clOQn8GHgFeLmqZpNcDnwG2A6cBP5ZVT2/sjIlSYtW44r7N6tqZ1XNtv27gENVtQM41PYlSatkLaZKdgP72/Z+4N1r8B6SNLVWGtwF/EWSp5LsbW1XVdVpgLa+coXvIUkasqI5buCGqjqV5ErgYJJvj3tiC/q9AFdfffUKy5Ck6bGiK+6qOtXWZ4DPA9cDzyXZDNDWZ85x7r6qmq2q2ZmZmZWUIUlTZdnBneSXkrxpcRv4beAocADY07rtAR5ZaZGSpF9YyVTJVcDnkyy+zn+rqv+Z5KvAw0luB74PvHflZUqSFi07uKvqu8BbR7T/b+AdKylKknRufnNSkjpjcEtSZwxuSeqMwS1JnTG4JakzBrckdcbglqTOGNyS1BmDW5I6Y3BLUmcMbknqjMEtSZ0xuCWpMwa3JHXG4JakzhjcktQZg1uSOmNwS1JnDG5J6sySwZ3kwSRnkhwdars8ycEkz7T1Za09ST6W5ESSI0netpbFS9I0GueK+xPAzWe13QUcqqodwKG2D/AuYEdb9gL3r06ZkqRFSwZ3Vf0l8MOzmncD+9v2fuDdQ+2frIGvAJcm2bxaxUqSlj/HfVVVnQZo6ytb+xbg2aF+863tVZLsTTKXZG5hYWGZZUjS9Fntm5MZ0VajOlbVvqqararZmZmZVS5Dkjau5Qb3c4tTIG19prXPA9uG+m0FTi2/PEnS2ZYb3AeAPW17D/DIUPv72tMlu4AXF6dUJEmr46KlOiT5NHAjcEWSeeAPgT8CHk5yO/B94L2t+2PALcAJ4KfA+9egZkmaaksGd1Xddo5D7xjRt4A7V1qUJOnc/OakJHXG4JakzhjcktQZg1uSOmNwS1JnDG5J6ozBLUmdMbglqTMGtyR1xuCWpM4Y3JLUGYNbkjpjcEtSZwxuSeqMwS1JnTG4JakzBrckdcbglqTOLBncSR5McibJ0aG2e5L8dZLDbbll6NjdSU4keTrJ76xV4ZI0rca54v4EcPOI9vuqamdbHgNIci1wK/Dr7Zz/nGTTahUrSRojuKvqL4Efjvl6u4GHquqlqvoeg197v34F9UmSzrKSOe4PJDnSplIua21bgGeH+sy3tldJsjfJXJK5hYWFFZQhSdNlucF9P/CrwE7gNPAnrT0j+taoF6iqfVU1W1WzMzMzyyxDkqbPsoK7qp6rqleq6mfAn/GL6ZB5YNtQ163AqZWVKEkatqzgTrJ5aPf3gMUnTg4Atya5JMk1wA7gyZWVKEkadtFSHZJ8GrgRuCLJPPCHwI1JdjKYBjkJ3AFQVceSPAx8C3gZuLOqXlmb0iVpOi0Z3FV124jmB87T/17g3pUUJUk6N785KUmdMbglqTMGtyR1xuCWpM4Y3JLUGYNbkjpjcEtSZwxuaQlP7buDp/bdMekypJ9b8gs4kgaGw/sf7f34BCvRtPOKW5I6Y3BL5+EUidYjg1uSOmNwS1JnDG7pAnljUpNmcEtSZwxu6Ry8Man1yuCWpM4Y3JLUmSWDO8m2JI8nOZ7kWJIPtvbLkxxM8kxbX9bak+RjSU4kOZLkbWs9COm14o1JrQfjXHG/DHy4qt4C7ALuTHItcBdwqKp2AIfaPsC7GPy6+w5gL3D/qlctrTHnt7WeLRncVXW6qr7Wtn8MHAe2ALuB/a3bfuDdbXs38Mka+ApwaZLNq165JE2pC5rjTrIduA54Ariqqk7DINyBK1u3LcCzQ6fNt7azX2tvkrkkcwsLCxdeuSRNqbGDO8kbgc8CH6qqH52v64i2elVD1b6qmq2q2ZmZmXHLkCbG+W2tF2MFd5KLGYT2p6rqc635ucUpkLY+09rngW1Dp28FTq1OuZKkcZ4qCfAAcLyqPjp06ACwp23vAR4Zan9fe7pkF/Di4pSK1INRNya92tZ6Ms4PKdwA/D7wzSSHW9sfAH8EPJzkduD7wHvbsceAW4ATwE+B969qxZI05ZYM7qr6MqPnrQHeMaJ/AXeusC5pInwMUD3wm5OS1BmDW5I6Y3BLS/DGpNYbg1uSOmNwS403JtULg1uSOmNwS1JnDG7pPLwxqfXI4JZwflt9MbglqTMGtyR1xuCWzsH5ba1XBrckdcbg1tTzxqR6Y3BLUmcMbknqjMEtjeCNSa1nBremmvPb6tE4Pxa8LcnjSY4nOZbkg639niR/neRwW24ZOufuJCeSPJ3kd9ZyAJI0bcb5seCXgQ9X1deSvAl4KsnBduy+qvoPw52TXAvcCvw68HeA/5Xk71XVK6tZuCRNqyWvuKvqdFV9rW3/GDgObDnPKbuBh6rqpar6HoNfe79+NYqVXgvOb2u9u6A57iTbgeuAJ1rTB5IcSfJgksta2xbg2aHT5jl/0EuSLsDYwZ3kjcBngQ9V1Y+A+4FfBXYCp4E/Wew64vQa8Xp7k8wlmVtYWLjgwqWVGnVj0qtt9WCs4E5yMYPQ/lRVfQ6gqp6rqleq6mfAn/GL6ZB5YNvQ6VuBU2e/ZlXtq6rZqpqdmZlZyRgkaaqM81RJgAeA41X10aH2zUPdfg842rYPALcmuSTJNcAO4MnVK1mSpts4T5XcAPw+8M0kh1vbHwC3JdnJYBrkJHAHQFUdS/Iw8C0GT6Tc6RMlkrR6lgzuqvoyo+etHzvPOfcC966gLmlN+cUb9cxvTkqNNybVC4NbU8erbfXO4JakzhjcktQZg1uSOmNwS3hjUn0xuDVVvDGpjcDglqTOGNyS1BmDW1PP+W31xuCWpM4Y3Joa3pjURmFwS1JnDG51K8kFLaPM3rFv2edKk2JwS1JnxvkhBal7cx/fy6On9/6Ntt/dvG9C1Ugr4xW3psLZoS31zODW1DLM1atxfiz49UmeTPKNJMeSfKS1X5PkiSTPJPlMkte19kva/ol2fPvaDkFannvumZ10CdKyjHPF/RJwU1W9FdgJ3JxkF/DHwH1VtQN4Hri99b8deL6qfg24r/WTJsr5bG0k4/xYcAE/absXt6WAm4B/3tr3A/cA9wO72zbAnwP/MUna60gTMXvHPsDw1sYw1hx3kk1JDgNngIPAd4AXqurl1mUe2NK2twDPArTjLwJvXs2iJWmajRXcVfVKVe0EtgLXA28Z1a2tR31b4VVX20n2JplLMrewsDBuvZI09S7oqZKqegH4ErALuDTJ4lTLVuBU254HtgG0478M/HDEa+2rqtmqmp2ZmVle9ZI0hcZ5qmQmyaVt+w3AO4HjwOPAe1q3PcAjbftA26cd/6Lz25K0esb55uRmYH+STQyC/uGqejTJt4CHkvx74OvAA63/A8B/TXKCwZX2rWtQtyRNrXGeKjkCXDei/bsM5rvPbv8/wHtXpTpJ0qv4zUlJ6ozBLUmdMbglqTP+s67qlg8raVp5xS1JnTG4JakzBrckdcbglqTOGNyS1BmDW5I6Y3BLUmcMbknqjMEtSZ0xuCWpMwa3JHXG4JakzhjcktQZg1uSOjPOjwW/PsmTSb6R5FiSj7T2TyT5XpLDbdnZ2pPkY0lOJDmS5G1rPQhJmibj/HvcLwE3VdVPklwMfDnJ/2jH/lVV/flZ/d8F7GjL24H721qStAqWvOKugZ+03Yvbcr5/wX438Ml23leAS5NsXnmpkiQYc447yaYkh4EzwMGqeqIdurdNh9yX5JLWtgV4duj0+dYmSVoFYwV3Vb1SVTuBrcD1Sf4BcDfw94F/DFwO/JvWPaNe4uyGJHuTzCWZW1hYWFbxkjSNLuipkqp6AfgScHNVnW7TIS8B/wW4vnWbB7YNnbYVODXitfZV1WxVzc7MzCyreEmaRuM8VTKT5NK2/QbgncC3F+etkwR4N3C0nXIAeF97umQX8GJVnV6T6iVpCo3zVMlmYH+STQyC/uGqejTJF5PMMJgaOQz8y9b/MeAW4ATwU+D9q1+2JE2vJYO7qo4A141ov+kc/Qu4c+WlSZJG8ZuTktQZg1uSOmNwS1JnDG5J6ozBLUmdMbglqTMGtyR1xuCWpM4Y3JLUGYNbkjpjcEtSZwxuSeqMwS1JnTG4JakzBrckdcbglqTOGNyS1BmDW5I6Y3BLUmcMbknqjMEtSZ0xuCWpM6mqSddAkh8DT0+6jjVyBfCDSRexBjbquGDjjs1x9eXvVtXMqAMXvdaVnMPTVTU76SLWQpK5jTi2jTou2Lhjc1wbh1MlktQZg1uSOrNegnvfpAtYQxt1bBt1XLBxx+a4Noh1cXNSkjS+9XLFLUka08SDO8nNSZ5OciLJXZOu50IleTDJmSRHh9ouT3IwyTNtfVlrT5KPtbEeSfK2yVV+fkm2JXk8yfEkx5J8sLV3PbYkr0/yZJJvtHF9pLVfk+SJNq7PJHlda7+k7Z9ox7dPsv6lJNmU5OtJHm37G2VcJ5N8M8nhJHOtrevP4kpMNLiTbAL+E/Au4FrgtiTXTrKmZfgEcPNZbXcBh6pqB3Co7cNgnDvashe4/zWqcTleBj5cVW8BdgF3tv82vY/tJeCmqnorsBO4Ocku4I+B+9q4ngdub/1vB56vql8D7mv91rMPAseH9jfKuAB+s6p2Dj361/tncfmqamIL8BvAF4b27wbunmRNyxzHduDo0P7TwOa2vZnBc+oAHwduG9VvvS/AI8BvbaSxAX8b+BrwdgZf4Liotf/8cwl8AfiNtn1R65dJ136O8WxlEGA3AY8C2QjjajWeBK44q23DfBYvdJn0VMkW4Nmh/fnW1rurquo0QFtf2dq7HG/7a/R1wBNsgLG16YTDwBngIPAd4IWqerl1Ga795+Nqx18E3vzaVjy2PwX+NfCztv9mNsa4AAr4iyRPJdnb2rr/LC7XpL85mRFtG/kxl+7Gm+SNwGeBD1XVj5JRQxh0HdG2LsdWVa8AO5NcCnweeMuobm3dxbiS/C5wpqqeSnLjYvOIrl2Na8gNVXUqyZXAwSTfPk/f3sZ2wSZ9xT0PbBva3wqcmlAtq+m5JJsB2vpMa+9qvEkuZhDan6qqz7XmDTE2gKp6AfgSgzn8S5MsXsgM1/7zcbXjvwz88LWtdCw3AP80yUngIQbTJX9K/+MCoKpOtfUZBn/YXs8G+ixeqEkH91eBHe3O9+uAW4EDE65pNRwA9rTtPQzmhxfb39fueu8CXlz8q956k8Gl9QPA8ar66NChrseWZKZdaZPkDcA7GdzMexx4T+t29rgWx/se4IvVJk7Xk6q6u6q2VtV2Bv8ffbGq/gWdjwsgyS8ledPiNvDbwFE6/yyuyKQn2YFbgL9iMM/4byddzzLq/zRwGvh/DP6kv53BXOEh4Jm2vrz1DYOnaL4DfBOYnXT95xnXP2Hw18sjwOG23NL72IB/CHy9jeso8O9a+68ATwIngP8OXNLaX9/2T7TjvzLpMYwxxhuBRzfKuNoYvtGWY4s50ftncSWL35yUpM5MeqpEknSBDG5J6ozBLUmdMbglqTMGtyR1xuCWpM4Y3JLUGYNbkjrz/wHXCBj9lJ9HGwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img=env.render('rgb_array')\n",
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400, 600, 3)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames=[]\n",
    "n_max_steps=10000\n",
    "n_change_steps=10\n",
    "obs=env.reset()\n",
    "for step in range(n_max_steps):\n",
    "    img=render_cart_pole(env,obs)\n",
    "    frames.append(img)\n",
    "    position,velocity,angle,angular_velocity=obs\n",
    "    if angle<0:\n",
    "        action=0\n",
    "    else:\n",
    "        action=1\n",
    "    \n",
    "    obs,reward,done,info=env.step(action)\n",
    "    if done:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0531 21:16:38.132002 140482778924864 deprecation.py:323] From <ipython-input-38-8244fdeddf55>:7: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "W0531 21:16:38.529641 140482778924864 deprecation.py:323] From <ipython-input-38-8244fdeddf55>:10: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.random.categorical` instead.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "n_inputs=4\n",
    "n_hidden=4\n",
    "n_outputs=1\n",
    "initializer=tf.variance_scaling_initializer()\n",
    "X=tf.placeholder(tf.float32,shape=[None,n_inputs])\n",
    "hidden=tf.layers.dense(X,n_hidden,activation=tf.nn.elu,kernel_initializer=initializer)\n",
    "outputs=tf.layers.dense(hidden,n_outputs,activation=tf.nn.sigmoid,kernel_initializer=initializer)\n",
    "p_left_and_right = tf.concat(axis=1, values=[outputs, 1 - outputs])\n",
    "action = tf.multinomial(tf.log(p_left_and_right), num_samples=1)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, None, 4)           20        \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, None, 1)           5         \n",
      "=================================================================\n",
      "Total params: 25\n",
      "Trainable params: 25\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dense\n",
    "from keras.models import Sequential\n",
    "model=Sequential()\n",
    "model.add(Dense(n_hidden,activation='elu',input_shape=(None,n_inputs)))\n",
    "model.add(Dense(n_outputs,activation='sigmoid'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'sequential_1/dense_2/Sigmoid:0' shape=(1, 1) dtype=float32>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(tf.convert_to_tensor(obs.reshape(1,n_inputs),dtype='float32'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_max_steps = 1000\n",
    "frames = []\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    obs = env.reset()\n",
    "    for step in range(n_max_steps):\n",
    "        img = render_cart_pole(env, obs)\n",
    "        frames.append(img)\n",
    "        action_val = action.eval(feed_dict={X: obs.reshape(1, n_inputs)})\n",
    "        obs, reward, done, info = env.step(action_val[0][0])\n",
    "        if done:\n",
    "            break\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fc460285c90>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAARv0lEQVR4nO3dYYxdZ33n8e+vTghsqZqETCLLdtZpa2mTVsWhs8FV9kUItDVRtaYSrJKtioUiDZWCBBVqN9lKW5A2UrtSSYV2G+EqWUxFCdkCihVlSyMTVPGChAGMsTFpDFhkaiseShJAqNlN+O+LeYbcda491zNzPfPM/X6ko3POc55z7/9Rbn4+fu45vqkqJEn9+Jm1LkCSdH4MbknqjMEtSZ0xuCWpMwa3JHXG4JakzowtuJPsTvJkkuNJ7hzX+0jSpMk47uNOsgn4R+A3gDngS8BtVfWNVX8zSZow47rivgE4XlXfrqr/AzwA7BnTe0nSRLloTK+7BXh6YH8OeOPZOl9xxRW1ffv2MZUiSf05ceIE3/ve9zLs2LiCe9ib/X9zMklmgBmAq6++mtnZ2TGVIkn9mZ6ePuuxcU2VzAHbBva3AicHO1TVvqqarqrpqampMZUhSRvPuIL7S8COJNckeRVwK3BgTO8lSRNlLFMlVfVikvcAnwU2AfdX1dFxvJckTZpxzXFTVY8Aj4zr9SVpUvnkpCR1xuCWpM4Y3JLUGYNbkjpjcEtSZwxuSeqMwS1JnTG4JakzBrckdcbglqTOGNyS1BmDW5I6Y3BLUmcMbknqjMEtSZ0xuCWpMwa3JHXG4Jakzqzop8uSnAB+CLwEvFhV00kuBz4JbAdOAP+hqp5dWZmSpEWrccX9pqraWVXTbf9O4GBV7QAOtn1J0ioZx1TJHmB/294PvG0M7yFJE2ulwV3A3yf5cpKZ1nZVVZ0CaOsrV/gekqQBK5rjBm6sqpNJrgQeTfLNUU9sQT8DcPXVV6+wDEmaHCu64q6qk219GvgMcAPwTJLNAG19+izn7quq6aqanpqaWkkZkjRRlh3cSX42yc8tbgO/CRwBDgB7W7e9wEMrLVKS9LKVTJVcBXwmyeLr/E1V/V2SLwEPJrkd+C7wjpWXKUlatOzgrqpvA68f0v7PwJtXUpQk6ex8clKSOmNwS1JnDG5J6ozBLUmdMbglqTMGtyR1xuCWpM4Y3JLUGYNbkjpjcEtSZwxuSeqMwS1JnTG4JakzBrckdcbglqTOGNyS1BmDW5I6Y3BLUmcMbknqzJLBneT+JKeTHBlouzzJo0meauvLWnuSfDjJ8SSHk7xhnMVL0iQa5Yr7o8DuM9ruBA5W1Q7gYNsHeCuwoy0zwL2rU6YkadGSwV1V/wB8/4zmPcD+tr0feNtA+8dqwReBS5NsXq1iJUnLn+O+qqpOAbT1la19C/D0QL+51vYKSWaSzCaZnZ+fX2YZkjR5VvvLyQxpq2Edq2pfVU1X1fTU1NQqlyFJG9dyg/uZxSmQtj7d2ueAbQP9tgInl1+eJOlMyw3uA8Detr0XeGig/Z3t7pJdwPOLUyqSpNVx0VIdknwCuAm4Iskc8CfAnwIPJrkd+C7wjtb9EeAW4DjwY+BdY6hZkibaksFdVbed5dCbh/Qt4I6VFiVJOjufnJSkzhjcktQZg1uSOmNwS1JnDG5J6ozBLUmdMbglqTMGtyR1xuCWpM4Y3JLUGYNbkjpjcEtSZwxuSeqMwS1JnTG4JakzBrckdcbglqTOGNyS1JklgzvJ/UlOJzky0PaBJP+U5FBbbhk4dleS40meTPJb4ypckibVKFfcHwV2D2m/p6p2tuURgCTXAbcCv9zO+cskm1arWEnSCMFdVf8AfH/E19sDPFBVL1TVd1j4tfcbVlCfJOkMK5njfk+Sw20q5bLWtgV4eqDPXGt7hSQzSWaTzM7Pz6+gDEmaLMsN7nuBXwR2AqeAP2/tGdK3hr1AVe2rqumqmp6amlpmGZI0eZYV3FX1TFW9VFU/Af6Kl6dD5oBtA123AidXVqIkadCygjvJ5oHd3wEW7zg5ANya5JIk1wA7gCdWVqIkadBFS3VI8gngJuCKJHPAnwA3JdnJwjTICeDdAFV1NMmDwDeAF4E7quql8ZQuSZNpyeCuqtuGNN93jv53A3evpChJ0tn55KQkdcbglqTOGNyS1BmDW5I6Y3BLUmcMbknqjMEtSZ1Z8j5uaSP78r53D23/tZmPXOBKpNF5xa2JZkCrRwa3JHXG4Jakzhjc0hBnm/uW1gODW5I6Y3BLUmcMbk087yxRbwxuSeqMwS1JnTG4pbPwzhKtV0sGd5JtSR5LcizJ0STvbe2XJ3k0yVNtfVlrT5IPJzme5HCSN4x7EJI0SUa54n4ReH9VXQvsAu5Ich1wJ3CwqnYAB9s+wFtZ+HX3HcAMcO+qVy1JE2zJ4K6qU1X1lbb9Q+AYsAXYA+xv3fYDb2vbe4CP1YIvApcm2bzqlUuryDtL1JPzmuNOsh24HngcuKqqTsFCuANXtm5bgKcHTptrbWe+1kyS2SSz8/Pz51+5JE2okYM7yWuBTwHvq6ofnKvrkLZ6RUPVvqqarqrpqampUcuQLii/oNR6NFJwJ7mYhdD+eFV9ujU/szgF0tanW/scsG3g9K3AydUpV5I0yl0lAe4DjlXVhwYOHQD2tu29wEMD7e9sd5fsAp5fnFKRJK3cKL+AcyPwe8DXkxxqbf8Z+FPgwSS3A98F3tGOPQLcAhwHfgy8a1UrlqQJt2RwV9UXGD5vDfDmIf0LuGOFdUkX3K/NfMQ5bXXBJyclqTMGt7QEr8K13hjcktQZg1uSOmNwSwN89F09MLglqTMGtyR1xuCWzjBsusQ7S7SeGNyS1BmDW5I6Y3BLI3K6ROuFwS1JnTG4JakzBrc0hA/iaD0zuCWpMwa3dB78glLrgcEtSZ0xuCWpM6P8WPC2JI8lOZbkaJL3tvYPJPmnJIfacsvAOXclOZ7kySS/Nc4BSNKkGeWK+0Xg/VV1LbALuCPJde3YPVW1sy2PALRjtwK/DOwG/jLJpjHULo2Vd5ZovVoyuKvqVFV9pW3/EDgGbDnHKXuAB6rqhar6Dgu/9n7DahQrSTrPOe4k24Hrgcdb03uSHE5yf5LLWtsW4OmB0+Y4d9BLXfHOEq21kYM7yWuBTwHvq6ofAPcCvwjsBE4Bf77YdcjpNeT1ZpLMJpmdn58/78IlaVKNFNxJLmYhtD9eVZ8GqKpnquqlqvoJ8Fe8PB0yB2wbOH0rcPLM16yqfVU1XVXTU1NTKxmDJE2UUe4qCXAfcKyqPjTQvnmg2+8AR9r2AeDWJJckuQbYATyxeiVL0mQb5Yr7RuD3gJvPuPXvvyX5epLDwJuAPwCoqqPAg8A3gL8D7qiql8ZTvjRe3lmi9eiipTpU1RcYPm/9yDnOuRu4ewV1SZLOwicnpWXwzhKtJYNbkjpjcEtSZwxuaQl+Qan1xuCWpM4Y3JLUGYNbWibvLNFaMbglqTMGtyR1xuCWRuCdJVpPDG5J6ozBLa2AX1BqLRjc0oicLtF6YXBLQJKRlpWce67XkM6HwS2t0OxHZta6BE2YJf89bkmv9PCpl8P6tzfvW8NKNIm84pbO02BoD9uXxs3glqTOjPJjwa9O8kSSryU5muSDrf2aJI8neSrJJ5O8qrVf0vaPt+PbxzsE6cKZfrfTIlp7o1xxvwDcXFWvB3YCu5PsAv4MuKeqdgDPAre3/rcDz1bVLwH3tH7ShnHmnLZz3LrQRvmx4AJ+1HYvbksBNwP/sbXvBz4A3AvsadsAfwv89yRpryN1b+Gq++Ww/sCaVaJJNdIcd5JNSQ4Bp4FHgW8Bz1XVi63LHLClbW8BngZox58HXreaRUvSJBspuKvqparaCWwFbgCuHdatrYc9YfCKq+0kM0lmk8zOz8+PWq8kTbzzuqukqp4DPg/sAi5NsjjVshU42bbngG0A7fjPA98f8lr7qmq6qqanpqaWV70kTaBR7iqZSnJp234N8BbgGPAY8PbWbS/wUNs+0PZpxz/n/LYkrZ5RnpzcDOxPsomFoH+wqh5O8g3ggST/FfgqcF/rfx/w10mOs3ClfesY6pakiTXKXSWHgeuHtH+bhfnuM9v/BXjHqlQnSXoFn5yUpM4Y3JLUGYNbkjrjP+sqAd74pJ54xS1JnTG4JakzBrckdcbglqTOGNyS1BmDW5I6Y3BLUmcMbknqjMEtSZ0xuCWpMwa3JHXG4JakzhjcktQZg1uSOjPKjwW/OskTSb6W5GiSD7b2jyb5TpJDbdnZ2pPkw0mOJzmc5A3jHoQkTZJR/j3uF4Cbq+pHSS4GvpDkf7djf1hVf3tG/7cCO9ryRuDetpYkrYIlr7hrwY/a7sVtOde/Or8H+Fg774vApUk2r7xUSRKMOMedZFOSQ8Bp4NGqerwdurtNh9yT5JLWtgV4euD0udYmSVoFIwV3Vb1UVTuBrcANSX4FuAv4N8C/BS4H/lPrnmEvcWZDkpkks0lm5+fnl1W8JE2i87qrpKqeAz4P7K6qU2065AXgfwI3tG5zwLaB07YCJ4e81r6qmq6q6ampqWUVL0mTaJS7SqaSXNq2XwO8Bfjm4rx1kgBvA460Uw4A72x3l+wCnq+qU2OpXpIm0Ch3lWwG9ifZxELQP1hVDyf5XJIpFqZGDgG/3/o/AtwCHAd+DLxr9cuWpMm1ZHBX1WHg+iHtN5+lfwF3rLw0SdIwPjkpSZ0xuCWpMwa3JHXG4JakzhjcktQZg1uSOmNwS1JnDG5J6ozBLUmdMbglqTMGtyR1xuCWpM4Y3JLUGYNbkjpjcEtSZwxuSeqMwS1JnTG4JakzBrckdcbglqTOGNyS1BmDW5I6k6pa6xpI8kPgybWuY0yuAL631kWMwUYdF2zcsTmuvvzrqpoaduCiC13JWTxZVdNrXcQ4JJndiGPbqOOCjTs2x7VxOFUiSZ0xuCWpM+sluPetdQFjtFHHtlHHBRt3bI5rg1gXX05Kkka3Xq64JUkjWvPgTrI7yZNJjie5c63rOV9J7k9yOsmRgbbLkzya5Km2vqy1J8mH21gPJ3nD2lV+bkm2JXksybEkR5O8t7V3PbYkr07yRJKvtXF9sLVfk+TxNq5PJnlVa7+k7R9vx7evZf1LSbIpyVeTPNz2N8q4TiT5epJDSWZbW9efxZVY0+BOsgn4H8BbgeuA25Jct5Y1LcNHgd1ntN0JHKyqHcDBtg8L49zRlhng3gtU43K8CLy/qq4FdgF3tP82vY/tBeDmqno9sBPYnWQX8GfAPW1czwK3t/63A89W1S8B97R+69l7gWMD+xtlXABvqqqdA7f+9f5ZXL6qWrMF+HXgswP7dwF3rWVNyxzHduDIwP6TwOa2vZmF+9QBPgLcNqzfel+Ah4Df2EhjA/4V8BXgjSw8wHFRa//p5xL4LPDrbfui1i9rXftZxrOVhQC7GXgYyEYYV6vxBHDFGW0b5rN4vstaT5VsAZ4e2J9rbb27qqpOAbT1la29y/G2v0ZfDzzOBhhbm044BJwGHgW+BTxXVS+2LoO1/3Rc7fjzwOsubMUj+wvgj4CftP3XsTHGBVDA3yf5cpKZ1tb9Z3G51vrJyQxp28i3uXQ33iSvBT4FvK+qfpAMG8JC1yFt63JsVfUSsDPJpcBngGuHdWvrLsaV5LeB01X15SQ3LTYP6drVuAbcWFUnk1wJPJrkm+fo29vYzttaX3HPAdsG9rcCJ9eoltX0TJLNAG19urV3Nd4kF7MQ2h+vqk+35g0xNoCqeg74PAtz+JcmWbyQGaz9p+Nqx38e+P6FrXQkNwL/PskJ4AEWpkv+gv7HBUBVnWzr0yz8YXsDG+izeL7WOri/BOxo33y/CrgVOLDGNa2GA8Detr2XhfnhxfZ3tm+9dwHPL/5Vb73JwqX1fcCxqvrQwKGux5Zkql1pk+Q1wFtY+DLvMeDtrduZ41oc79uBz1WbOF1PququqtpaVdtZ+P/oc1X1u3Q+LoAkP5vk5xa3gd8EjtD5Z3FF1nqSHbgF+EcW5hn/eK3rWUb9nwBOAf+XhT/pb2dhrvAg8FRbX976hoW7aL4FfB2YXuv6zzGuf8fCXy8PA4fackvvYwN+FfhqG9cR4L+09l8AngCOA/8LuKS1v7rtH2/Hf2GtxzDCGG8CHt4o42pj+Fpbji7mRO+fxZUsPjkpSZ1Z66kSSdJ5MrglqTMGtyR1xuCWpM4Y3JLUGYNbkjpjcEtSZwxuSerM/wN8nxG/qAULAwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "action_val[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# To plot pretty figures and animations\n",
    "%matplotlib nbagg\n",
    "import matplotlib\n",
    "import matplotlib.animation as animation\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"rl\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0531 21:17:27.531992 140482778924864 deprecation.py:323] From /home/rahul/miniconda3/lib/python3.7/site-packages/tensorflow/python/ops/nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "n_inputs = 4\n",
    "n_hidden = 4\n",
    "n_outputs = 1\n",
    "learning_rate=0.01\n",
    "\n",
    "initializer=tf.variance_scaling_initializer()\n",
    "\n",
    "X=tf.placeholder(tf.float32,shape=[None,n_inputs])\n",
    "y=tf.placeholder(tf.float32,shape=[None,n_outputs])\n",
    "\n",
    "hidden=tf.layers.dense(X,n_hidden,activation=tf.nn.elu,kernel_initializer=initializer)\n",
    "logits=tf.layers.dense(hidden,n_outputs)\n",
    "outputs=tf.nn.sigmoid(logits)\n",
    "p_left_and_right=tf.concat(axis=1,values=[outputs,1-outputs])\n",
    "action=tf.multinomial(tf.log(p_left_and_right),num_samples=1)\n",
    "\n",
    "cross_entropy=tf.nn.sigmoid_cross_entropy_with_logits(labels=y,logits=logits)\n",
    "optimizer=tf.train.AdamOptimizer(learning_rate)\n",
    "training_op=optimizer.minimize(cross_entropy)\n",
    "\n",
    "init=tf.global_variables_initializer()\n",
    "saver=tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Attempted to use a closed Session.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-52-c42f54c83f89>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0mobservations\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0menv_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobs\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"./my_policy_net_basic.cpkt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0menv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menvs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, sess, save_path, global_step, latest_filename, meta_graph_suffix, write_meta_graph, write_state, strip_default_attrs, save_debug_info)\u001b[0m\n\u001b[1;32m   1171\u001b[0m           model_checkpoint_path = sess.run(\n\u001b[1;32m   1172\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaver_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_tensor_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1173\u001b[0;31m               {self.saver_def.filename_tensor_name: checkpoint_file})\n\u001b[0m\u001b[1;32m   1174\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m         \u001b[0mmodel_checkpoint_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_checkpoint_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1094\u001b[0m     \u001b[0;31m# Check session.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1095\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_closed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1096\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Attempted to use a closed Session.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1097\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mversion\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1098\u001b[0m       raise RuntimeError('The Session graph is empty.  Add operations to the '\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Attempted to use a closed Session."
     ]
    }
   ],
   "source": [
    "n_enviroments=10\n",
    "n_iterations=1000\n",
    "\n",
    "envs=[gym.make('CartPole-v0') for _ in range(n_enviroments)]\n",
    "observations=[env.reset() for env in envs]\n",
    "\n",
    "with tf.Session() as Sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        target_probas=np.array([([1.] if obs[2]<0 else [0.]) for obs in observations])\n",
    "        action_val,_=Sess.run([action,training_op],feed_dict={X:np.array(observations),y:target_probas})\n",
    "        for env_index,env in enumerate(envs):\n",
    "            obs,reward,done,info=env.step(action_val[env_index][0])\n",
    "            observations[env_index]=obs if not done else env.reset()\n",
    "            \n",
    "    saver.save(sess,\"./my_policy_net_basic.cpkt\")\n",
    "for env in envs:\n",
    "    env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'universe'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-57-67b44827881d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0muniverse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'universe'"
     ]
    }
   ],
   "source": [
    "import universe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "ename": "UnregisteredEnv",
     "evalue": "No registered env with id: Tetris-v0",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mspec\u001b[0;34m(self, path)\u001b[0m\n\u001b[1;32m    117\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 118\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv_specs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    119\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Tetris-v0'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mUnregisteredEnv\u001b[0m                           Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-55-f99a8d20a365>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgym_tetris\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactions\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMOVEMENT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgym_tetris\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Tetris-v0'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mJoypadSpace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMOVEMENT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(id, **kwargs)\u001b[0m\n\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 142\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mregistry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m     84\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Making new env: %s'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m         \u001b[0mspec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m         \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m         \u001b[0;31m# We used to have people override _reset/_step rather than\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mspec\u001b[0;34m(self, path)\u001b[0m\n\u001b[1;32m    126\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDeprecatedEnv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Env {} not found (valid versions include {})'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatching_envs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mUnregisteredEnv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'No registered env with id: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mUnregisteredEnv\u001b[0m: No registered env with id: Tetris-v0"
     ]
    }
   ],
   "source": [
    "from nes_py.wrappers import JoypadSpace\n",
    "import gym_tetris\n",
    "from gym_tetris.actions import MOVEMENT\n",
    "\n",
    "env = gym_tetris.make('Tetris-v0')\n",
    "env = JoypadSpace(env, MOVEMENT)\n",
    "\n",
    "done = True\n",
    "for step in range(5000):\n",
    "    if done:\n",
    "        state = env.reset()\n",
    "    state, reward, done, info = env.step(env.action_space.sample())\n",
    "    env.render()\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i_episode in range(20):\n",
    "    observation = env.reset()\n",
    "    for t in range(100):\n",
    "        env.render()\n",
    "        print(observation)\n",
    "        action = env.action_space.sample()\n",
    "        observation, reward, done, info = env.step(action)\n",
    "        if done:\n",
    "            print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "            break\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(100):\n",
    "    obs=env.reset()\n",
    "    env.render()\n",
    "    action=env.action_space.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames=[]\n",
    "total_reward=[]\n",
    "max_iter=2000\n",
    "change_steps=10\n",
    "env.reset()\n",
    "for i in range(max_iter):\n",
    "    img=env.render(mode='rgb_array')\n",
    "    frames.append(img)\n",
    "    if i%change_steps==0:\n",
    "        action=env.action_space.sample()\n",
    "    obs,reward, done,info=env.step(action)\n",
    "    total_reward.append(reward)\n",
    "    if done:\n",
    "        break\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames=np.array(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout,Flatten\n",
    "model=Sequential()\n",
    "model.add(Dense(15,activation='elu',input_shape=(210, 160, 3)))\n",
    "model.add(Dense(10,activation='elu'))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1,activation='elu'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='sparse_categorical_crossentropy',optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.array(total_reward).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames=np.array(frames).reshape(200,210,160,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler=MinMaxScaler((0,0.99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_reward=scaler.fit_transform(pd.DataFrame(total_reward)).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history=model.fit(frames,total_reward,batch_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}