testing-custom cnn 1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "african-rebel",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch.utils.data import DataLoader\n",
    "from torch import nn\n",
    "\n",
    "import os\n",
    "import cv2\n",
    "from tqdm.notebook import trange\n",
    "\n",
    "import gym\n",
    "from gym import spaces\n",
    "\n",
    "from stable_baselines3 import DQN\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.dqn.policies import CnnPolicy, DQNPolicy\n",
    "from stable_baselines3.common.torch_layers import BaseFeaturesExtractor\n",
    "from stable_baselines3.common.preprocessing import get_flattened_obs_dim, is_image_space\n",
    "from typing import Any, Callable, Dict, List, NamedTuple, Tuple, Union, Optional, Type\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv\n",
    "import sklearn as sk\n",
    "from sklearn import datasets\n",
    "# from stable_baselines3.common.policies import register_policy\n",
    "# register_policy(\"myPolicy\", CnnPolicy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "agricultural-camcorder",
   "metadata": {},
   "outputs": [],
   "source": [
    "Schedule = Callable[[float], float]\n",
    "\n",
    "class CustomCNN(BaseFeaturesExtractor):\n",
    "    \"\"\"\n",
    "    CNN from DQN nature paper:\n",
    "        Mnih, Volodymyr, et al.\n",
    "        \"Human-level control through deep reinforcement learning.\"\n",
    "        Nature 518.7540 (2015): 529-533.\n",
    "    :param observation_space:\n",
    "    :param features_dim: Number of features extracted.\n",
    "        This corresponds to the number of unit for the last layer.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 512):\n",
    "        super(CustomCNN, self).__init__(observation_space, features_dim)\n",
    "        # We assume CxHxW images (channels first)\n",
    "        # Re-ordering will be done by pre-preprocessing or wrapper\n",
    "        assert is_image_space(observation_space), (\n",
    "            \"You should use NatureCNN \"\n",
    "            f\"only with images not with {observation_space}\\n\"\n",
    "            \"(you are probably using `CnnPolicy` instead of `MlpPolicy`)\\n\"\n",
    "            \"If you are using a custom environment,\\n\"\n",
    "            \"please check it using our env checker:\\n\"\n",
    "            \"https://stable-baselines3.readthedocs.io/en/master/common/env_checker.html\"\n",
    "        )\n",
    "        n_input_channels = observation_space.shape[0]\n",
    "        self.cnn = nn.Sequential(\n",
    "            nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=0.2),\n",
    "            nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=0.3),\n",
    "            nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0),\n",
    "            nn.ReLU(),\n",
    "            nn.Flatten(),\n",
    "        )\n",
    "\n",
    "#         self.cnn1 = nn.Sequential(\n",
    "#             nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0),\n",
    "#             nn.ReLU(),\n",
    "#             nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0),\n",
    "#             nn.ReLU(),\n",
    "#             nn.MaxPooling(2,stride=2),\n",
    "#             nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0),\n",
    "#             nn.Dropout(n=0.4),\n",
    "#             nn.ReLU(),\n",
    "#             nn.Flatten(),\n",
    "#         )\n",
    "\n",
    "        # Compute shape by doing one forward pass\n",
    "        with torch.no_grad():\n",
    "            n_flatten = self.cnn(torch.as_tensor(observation_space.sample()[None]).float()).shape[1]\n",
    "\n",
    "        self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim), nn.ReLU())\n",
    "\n",
    "    def forward(self, observations: torch.Tensor) -> torch.Tensor:\n",
    "        return self.linear(self.cnn(observations))\n",
    "\n",
    "# policy_kwargs = dict(\n",
    "#     features_extractor_class=CustomCNN,\n",
    "#     features_extractor_kwargs=dict(features_dim=64),\n",
    "# )\n",
    "# model = PPO(\"CnnPolicy\", \"BreakoutNoFrameskip-v4\", policy_kwargs=policy_kwargs, verbose=1)\n",
    "\n",
    "class CustomPolicy(DQNPolicy):\n",
    "    \"\"\"\n",
    "    Policy class for DQN when using images as input.\n",
    "    :param observation_space: Observation space\n",
    "    :param action_space: Action space\n",
    "    :param lr_schedule: Learning rate schedule (could be constant)\n",
    "    :param net_arch: The specification of the policy and value networks.\n",
    "    :param activation_fn: Activation function\n",
    "    :param features_extractor_class: Features extractor to use.\n",
    "    :param normalize_images: Whether to normalize images or not,\n",
    "         dividing by 255.0 (True by default)\n",
    "    :param optimizer_class: The optimizer to use,\n",
    "        ``torch.optim.Adam`` by default\n",
    "    :param optimizer_kwargs: Additional keyword arguments,\n",
    "        excluding the learning rate, to pass to the optimizer\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        observation_space: gym.spaces.Space,\n",
    "        action_space: gym.spaces.Space,\n",
    "        lr_schedule: Schedule,\n",
    "        net_arch: Optional[List[int]] = None,\n",
    "        activation_fn: Type[nn.Module] = nn.ReLU,\n",
    "        features_extractor_class: Type[BaseFeaturesExtractor] = CustomCNN,\n",
    "        features_extractor_kwargs: Optional[Dict[str, Any]] = None,\n",
    "        normalize_images: bool = True,\n",
    "        optimizer_class: Type[torch.optim.Optimizer] = torch.optim.Adam,\n",
    "        optimizer_kwargs: Optional[Dict[str, Any]] = None,\n",
    "    ):\n",
    "        super(CustomPolicy, self).__init__(\n",
    "            observation_space,\n",
    "            action_space,\n",
    "            lr_schedule,\n",
    "            net_arch,\n",
    "            activation_fn,\n",
    "            features_extractor_class,\n",
    "            features_extractor_kwargs,\n",
    "            normalize_images,\n",
    "            optimizer_class,\n",
    "            optimizer_kwargs,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "economic-oxide",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the gym environment.\n",
    "class USPSGym(gym.Env):\n",
    "    def __init__(self, dataset,width=128, height=128, channels=1):\n",
    "        # Training dataset (Handwritten digits on a 16x16px canvas).\n",
    "        self.X, self.y = dataset\n",
    "        \n",
    "        # Reset the state index, used to step through dataset.\n",
    "        self.idx = 0\n",
    "        \n",
    "        # Digits 0-9 are valid actions.\n",
    "        self.action_space = spaces.Discrete(10)\n",
    "        \n",
    "        # A 1-channel canvas is used for observations.\n",
    "        self.observation_space = spaces.Box(low=0, high=255, shape=(width, height, channels), dtype=np.uint8)\n",
    "    def _obs(self):\n",
    "        # Return a frame at the target dimensions from self.X at the current state index for the CnnPolicy.\n",
    "        width, height, channels = (self.observation_space.shape[0],\n",
    "                                   self.observation_space.shape[1],\n",
    "                                   self.observation_space.shape[2])\n",
    "        obs = self.X[self.idx]\n",
    "        \n",
    "        # Enlarge the observation if the dataset is smaller than the target canvas.\n",
    "        if obs.shape[0] < width or obs.shape[1] < height:\n",
    "            obs = cv2.resize(np.array(obs).astype(np.float32), (width, height), interpolation = cv2.INTER_CUBIC)\n",
    "            obs = obs.reshape(width, height, channels)\n",
    "        return obs\n",
    "    def step(self, action):\n",
    "        # The agent earns 1 point for a correct label.\n",
    "        reward = 1 if action == self.y[self.idx] else 0\n",
    "        \n",
    "        # The state index increments at each step then wraps around at the end of the training dataset.\n",
    "        self.idx = self.idx + 1 if self.idx < len(self.X) - 1 else 0\n",
    "        \n",
    "        # Return the observation, earned reward, terminal state, and info dict.\n",
    "        return self._obs(), reward, self.idx == 0, {}\n",
    "    def reset(self):\n",
    "        # Reset the index to the beginning of the training dataset and return the initial observation.\n",
    "        self.idx = 0\n",
    "        return self._obs()\n",
    "    def render(self, action='', mode='human', close=False):\n",
    "        # Display the labeled observation.\n",
    "        width, height = self.observation_space.shape[0], self.observation_space.shape[1]\n",
    "        fig, ax = plt.subplots(1)\n",
    "        ax.imshow(self._obs().reshape(width, height), cmap='Greys')\n",
    "        \n",
    "        # Label with the correct value and action if supplied. \n",
    "        title = '{}-{}'.format(action, self.y[self.idx]) if action != '' else self.y[self.idx]\n",
    "        ax.set_title(title)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "likely-check",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_svmlight_file\n",
    "from dqn import *\n",
    "import numpy as np\n",
    "from stable_baselines3 import DQN\n",
    "\n",
    "\n",
    "train = load_svmlight_file('data/mmd_mnist_torch')\n",
    "x, y = train\n",
    "x = x.todense()\n",
    "\n",
    "sortind = np.argsort(y)\n",
    "x = x[sortind, :]\n",
    "y = y[sortind]\n",
    "\n",
    "test = load_svmlight_file('data/mmd_mnist_torch.t')\n",
    "testx, testy = test\n",
    "testx = testx.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "favorite-wednesday",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Complete data 7291\n",
    "# Load the custom gym into a vectorized environment.\n",
    "env = DummyVecEnv([lambda: USPSGym(width=64, height=64, channels=1, dataset=(x, y))])\n",
    "\n",
    "# Grab the observation shape for generating evaluation frames.\n",
    "width, height = env.observation_space.shape[0], env.observation_space.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "specialized-bottle",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title complete 60000 - create model\n",
    "def create_model(pretrained=False, save_model=True, epochs=2):\n",
    "    model_name = \"dqn_cnn_mnist_{}_custom_tarun.zip\".format(epochs)\n",
    "    \n",
    "    # Return a pretrained model if the flag is set. Otherwise, train a new model.\n",
    "    if pretrained:\n",
    "        return DQN.load(model_name)\n",
    "\n",
    "    # Create a model from a DQN agent with a CnnPolicy attached to a tensorboard logger.\n",
    "    \n",
    "    # Train the model on several epochs through the full training dataset.\n",
    "#     model = DQN(CnnPolicy, env, policy_kwargs=policy_kwargs, verbose=1)\n",
    "    model = DQN(CustomPolicy, env, verbose=1)\n",
    "    \n",
    "    model.learn(total_timesteps=len(x) * epochs)\n",
    "#     model.learn(total_timesteps=300000, log_interval=1)\n",
    "    \n",
    "    # Save the new model if the flag is set.\n",
    "    if save_model:\n",
    "        model.save(model_name)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "several-richards",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #@title eval\n",
    "# # Evaluate the model by counting the total rewards attained on the test dataset.\n",
    "# total_rewards = 0\n",
    "# pred_y_test = []\n",
    "# count = 0\n",
    "\n",
    "# for idx in trange(len(testx)):\n",
    "#     # Generate an evaluation observation frame.\n",
    "#     obs = cv2.resize(np.array(testx[idx]).astype(np.float32), (width, height), interpolation = cv2.INTER_CUBIC)\n",
    "#     obs = obs.reshape(width, height, 1)\n",
    "    \n",
    "#     # Predict an action based on the observation.\n",
    "#     action, _states = inter_model.predict(obs)\n",
    "#     pred_y_test.append(action)\n",
    "\n",
    "\n",
    "#     # Score the prediction.\n",
    "#     if (action.size > 1):\n",
    "#         # print(action)\n",
    "#         count += 1\n",
    "#         pass\n",
    "#     else:\n",
    "#         reward = 1 if action == testy[idx] else 0\n",
    "#         total_rewards += reward\n",
    "\n",
    "# print('Accuracy: {:.2f}%'.format(total_rewards / (len(testy) - count) * 100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "defensive-sheet",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17 \n",
      "\n",
      "Using cpu device\n",
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-70-63b43d749894>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m17\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0minter_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;31m#@title eval\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-66-7b014ee134fd>\u001b[0m in \u001b[0;36mcreate_model\u001b[0;34m(pretrained, save_model, epochs)\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDQN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCustomPolicy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;31m#     model.learn(total_timesteps=300000, log_interval=1)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/dqn/dqn.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    232\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0meval_log_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0meval_log_path\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m             \u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m         )\n\u001b[1;32m    236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    267\u001b[0m                 \u001b[0;31m# do as many gradients steps as steps performed during the rollout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m                 \u001b[0mgradient_steps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient_steps\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient_steps\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepisode_timesteps\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 269\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgradient_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    270\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    271\u001b[0m         \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_training_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/dqn/dqn.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, gradient_steps, batch_size)\u001b[0m\n\u001b[1;32m    178\u001b[0m             \u001b[0;31m# Clip gradient norm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m             \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpolicy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_grad_norm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpolicy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m         \u001b[0;31m# Increase update counter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    117\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'weight_decay'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m                    )\n\u001b[1;32m    121\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/torch/optim/functional.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps)\u001b[0m\n\u001b[1;32m     92\u001b[0m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmax_exp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0mstep_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "epochs = [1, 5, 10, 25, 50, 75, 100, 667]\n",
    "# for epoch in epochs[:1]:\n",
    "for epoch in [17]:\n",
    "    print(epoch, '\\n')\n",
    "    inter_model = create_model(pretrained=False, save_model=True, epochs=epoch)\n",
    "    \n",
    "    #@title eval\n",
    "    # Evaluate the model by counting the total rewards attained on the test dataset.\n",
    "    total_rewards = 0\n",
    "    pred_y_test = []\n",
    "    count = 0\n",
    "\n",
    "    for idx in trange(len(testx)):\n",
    "        # Generate an evaluation observation frame.\n",
    "        obs = cv2.resize(np.array(testx[idx]).astype(np.float32), (width, height), interpolation = cv2.INTER_CUBIC)\n",
    "        obs = obs.reshape(width, height, 1)\n",
    "\n",
    "        # Predict an action based on the observation.\n",
    "        action, _states = inter_model.predict(obs)\n",
    "        pred_y_test.append(action)\n",
    "\n",
    "\n",
    "        # Score the prediction.\n",
    "        if (action.size > 1):\n",
    "            # print(action)\n",
    "            count += 1\n",
    "            pass\n",
    "        else:\n",
    "            reward = 1 if action == testy[idx] else 0\n",
    "            total_rewards += reward\n",
    "\n",
    "    print('Accuracy: {:.2f}%'.format(total_rewards / (len(testy) - count) * 100.0))\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "likely-tackle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from glob import glob\n",
    "# import numpy as np\n",
    "\n",
    "# protos = sorted(glob('protos/*'))\n",
    "# critics = sorted(glob('critics/*'))\n",
    "# combined = zip(protos, critics)\n",
    "# combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ordered-robinson",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for p, c in combined:\n",
    "#     m = p.split('_')[-1].split('.')[0]\n",
    "#     npro = np.load(p)\n",
    "#     ncri = np.load(c)\n",
    "#     c = np.concatenate((npro, ncri))\n",
    "#     np.save('combined/combined_{}'.format(m), c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "incomplete-browser",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}