testing-custom cnn 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "african-rebel",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch.utils.data import DataLoader\n",
    "from torch import nn\n",
    "\n",
    "import os\n",
    "import cv2\n",
    "from tqdm.notebook import trange\n",
    "\n",
    "import gym\n",
    "from gym import spaces\n",
    "\n",
    "from stable_baselines3 import DQN\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.dqn.policies import CnnPolicy, DQNPolicy\n",
    "from stable_baselines3.common.torch_layers import BaseFeaturesExtractor\n",
    "from stable_baselines3.common.preprocessing import get_flattened_obs_dim, is_image_space\n",
    "from typing import Any, Callable, Dict, List, NamedTuple, Tuple, Union, Optional, Type\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv\n",
    "import sklearn as sk\n",
    "from sklearn import datasets\n",
    "# from stable_baselines3.common.policies import register_policy\n",
    "# register_policy(\"myPolicy\", CnnPolicy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "under-diabetes",
   "metadata": {},
   "outputs": [],
   "source": [
    "Schedule = Callable[[float], float]\n",
    "\n",
    "class CustomCNN(BaseFeaturesExtractor):\n",
    "    \"\"\"\n",
    "    CNN from DQN nature paper:\n",
    "        Mnih, Volodymyr, et al.\n",
    "        \"Human-level control through deep reinforcement learning.\"\n",
    "        Nature 518.7540 (2015): 529-533.\n",
    "    :param observation_space:\n",
    "    :param features_dim: Number of features extracted.\n",
    "        This corresponds to the number of unit for the last layer.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 512):\n",
    "        super(CustomCNN, self).__init__(observation_space, features_dim)\n",
    "        # We assume CxHxW images (channels first)\n",
    "        # Re-ordering will be done by pre-preprocessing or wrapper\n",
    "        assert is_image_space(observation_space), (\n",
    "            \"You should use NatureCNN \"\n",
    "            f\"only with images not with {observation_space}\\n\"\n",
    "            \"(you are probably using `CnnPolicy` instead of `MlpPolicy`)\\n\"\n",
    "            \"If you are using a custom environment,\\n\"\n",
    "            \"please check it using our env checker:\\n\"\n",
    "            \"https://stable-baselines3.readthedocs.io/en/master/common/env_checker.html\"\n",
    "        )\n",
    "        n_input_channels = observation_space.shape[0]\n",
    "#         self.cnn = nn.Sequential(\n",
    "#             nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0),\n",
    "#             nn.ReLU(),\n",
    "#             nn.Dropout(p=0.2),\n",
    "#             nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0),\n",
    "#             nn.ReLU(),\n",
    "#             nn.Dropout(p=0.3),\n",
    "#             nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0),\n",
    "#             nn.ReLU(),\n",
    "#             nn.Flatten(),\n",
    "#         )\n",
    "\n",
    "        self.cnn = nn.Sequential(\n",
    "            nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2,stride=2),\n",
    "            nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0),\n",
    "            nn.Dropout(p=0.4),\n",
    "            nn.ReLU(),\n",
    "            nn.Flatten(),\n",
    "        )\n",
    "\n",
    "        # Compute shape by doing one forward pass\n",
    "        with torch.no_grad():\n",
    "            n_flatten = self.cnn(torch.as_tensor(observation_space.sample()[None]).float()).shape[1]\n",
    "\n",
    "        self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim), nn.ReLU())\n",
    "\n",
    "    def forward(self, observations: torch.Tensor) -> torch.Tensor:\n",
    "        return self.linear(self.cnn(observations))\n",
    "\n",
    "# policy_kwargs = dict(\n",
    "#     features_extractor_class=CustomCNN,\n",
    "#     features_extractor_kwargs=dict(features_dim=64),\n",
    "# )\n",
    "# model = PPO(\"CnnPolicy\", \"BreakoutNoFrameskip-v4\", policy_kwargs=policy_kwargs, verbose=1)\n",
    "\n",
    "class CustomPolicy(DQNPolicy):\n",
    "    \"\"\"\n",
    "    Policy class for DQN when using images as input.\n",
    "    :param observation_space: Observation space\n",
    "    :param action_space: Action space\n",
    "    :param lr_schedule: Learning rate schedule (could be constant)\n",
    "    :param net_arch: The specification of the policy and value networks.\n",
    "    :param activation_fn: Activation function\n",
    "    :param features_extractor_class: Features extractor to use.\n",
    "    :param normalize_images: Whether to normalize images or not,\n",
    "         dividing by 255.0 (True by default)\n",
    "    :param optimizer_class: The optimizer to use,\n",
    "        ``torch.optim.Adam`` by default\n",
    "    :param optimizer_kwargs: Additional keyword arguments,\n",
    "        excluding the learning rate, to pass to the optimizer\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        observation_space: gym.spaces.Space,\n",
    "        action_space: gym.spaces.Space,\n",
    "        lr_schedule: Schedule,\n",
    "        net_arch: Optional[List[int]] = None,\n",
    "        activation_fn: Type[nn.Module] = nn.ReLU,\n",
    "        features_extractor_class: Type[BaseFeaturesExtractor] = CustomCNN,\n",
    "        features_extractor_kwargs: Optional[Dict[str, Any]] = None,\n",
    "        normalize_images: bool = True,\n",
    "        optimizer_class: Type[torch.optim.Optimizer] = torch.optim.Adam,\n",
    "        optimizer_kwargs: Optional[Dict[str, Any]] = None,\n",
    "    ):\n",
    "        super(CustomPolicy, self).__init__(\n",
    "            observation_space,\n",
    "            action_space,\n",
    "            lr_schedule,\n",
    "            net_arch,\n",
    "            activation_fn,\n",
    "            features_extractor_class,\n",
    "            features_extractor_kwargs,\n",
    "            normalize_images,\n",
    "            optimizer_class,\n",
    "            optimizer_kwargs,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "economic-oxide",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the gym environment.\n",
    "class USPSGym(gym.Env):\n",
    "    def __init__(self, dataset,width=128, height=128, channels=1):\n",
    "        # Training dataset (Handwritten digits on a 16x16px canvas).\n",
    "        self.X, self.y = dataset\n",
    "        \n",
    "        # Reset the state index, used to step through dataset.\n",
    "        self.idx = 0\n",
    "        \n",
    "        # Digits 0-9 are valid actions.\n",
    "        self.action_space = spaces.Discrete(10)\n",
    "        \n",
    "        # A 1-channel canvas is used for observations.\n",
    "        self.observation_space = spaces.Box(low=0, high=255, shape=(width, height, channels), dtype=np.uint8)\n",
    "    def _obs(self):\n",
    "        # Return a frame at the target dimensions from self.X at the current state index for the CnnPolicy.\n",
    "        width, height, channels = (self.observation_space.shape[0],\n",
    "                                   self.observation_space.shape[1],\n",
    "                                   self.observation_space.shape[2])\n",
    "        obs = self.X[self.idx]\n",
    "        \n",
    "        # Enlarge the observation if the dataset is smaller than the target canvas.\n",
    "        if obs.shape[0] < width or obs.shape[1] < height:\n",
    "            obs = cv2.resize(np.array(obs).astype(np.float32), (width, height), interpolation = cv2.INTER_CUBIC)\n",
    "            obs = obs.reshape(width, height, channels)\n",
    "        return obs\n",
    "    def step(self, action):\n",
    "        # The agent earns 1 point for a correct label.\n",
    "        reward = 1 if action == self.y[self.idx] else 0\n",
    "        \n",
    "        # The state index increments at each step then wraps around at the end of the training dataset.\n",
    "        self.idx = self.idx + 1 if self.idx < len(self.X) - 1 else 0\n",
    "        \n",
    "        # Return the observation, earned reward, terminal state, and info dict.\n",
    "        return self._obs(), reward, self.idx == 0, {}\n",
    "    def reset(self):\n",
    "        # Reset the index to the beginning of the training dataset and return the initial observation.\n",
    "        self.idx = 0\n",
    "        return self._obs()\n",
    "    def render(self, action='', mode='human', close=False):\n",
    "        # Display the labeled observation.\n",
    "        width, height = self.observation_space.shape[0], self.observation_space.shape[1]\n",
    "        fig, ax = plt.subplots(1)\n",
    "        ax.imshow(self._obs().reshape(width, height), cmap='Greys')\n",
    "        \n",
    "        # Label with the correct value and action if supplied. \n",
    "        title = '{}-{}'.format(action, self.y[self.idx]) if action != '' else self.y[self.idx]\n",
    "        ax.set_title(title)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "likely-check",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_svmlight_file\n",
    "from dqn import *\n",
    "import numpy as np\n",
    "from stable_baselines3 import DQN\n",
    "\n",
    "\n",
    "train = load_svmlight_file('data/mmd_mnist_torch')\n",
    "x, y = train\n",
    "x = x.todense()\n",
    "\n",
    "sortind = np.argsort(y)\n",
    "x = x[sortind, :]\n",
    "y = y[sortind]\n",
    "\n",
    "test = load_svmlight_file('data/mmd_mnist_torch.t')\n",
    "testx, testy = test\n",
    "testx = testx.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "favorite-wednesday",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Complete data 7291\n",
    "# Load the custom gym into a vectorized environment.\n",
    "env = DummyVecEnv([lambda: USPSGym(width=64, height=64, channels=1, dataset=(x, y))])\n",
    "\n",
    "# Grab the observation shape for generating evaluation frames.\n",
    "width, height = env.observation_space.shape[0], env.observation_space.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "specialized-bottle",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title complete 60000 - create model\n",
    "def create_model(pretrained=False, save_model=True, epochs=2):\n",
    "    model_name = \"dqn_cnn_mnist_{}_custom_2_tarun.zip\".format(epochs)\n",
    "    \n",
    "    # Return a pretrained model if the flag is set. Otherwise, train a new model.\n",
    "    if pretrained:\n",
    "        return DQN.load(model_name)\n",
    "\n",
    "    # Create a model from a DQN agent with a CnnPolicy attached to a tensorboard logger.\n",
    "    \n",
    "    # Train the model on several epochs through the full training dataset.\n",
    "#     model = DQN(CnnPolicy, env, policy_kwargs=policy_kwargs, verbose=1)\n",
    "    model = DQN(CustomPolicy, env, verbose=1)\n",
    "    \n",
    "    model.learn(total_timesteps=len(x) * epochs)\n",
    "#     model.learn(total_timesteps=300000, log_interval=1)\n",
    "    \n",
    "    # Save the new model if the flag is set.\n",
    "    if save_model:\n",
    "        model.save(model_name)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "several-richards",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #@title eval\n",
    "# # Evaluate the model by counting the total rewards attained on the test dataset.\n",
    "# total_rewards = 0\n",
    "# pred_y_test = []\n",
    "# count = 0\n",
    "\n",
    "# for idx in trange(len(testx)):\n",
    "#     # Generate an evaluation observation frame.\n",
    "#     obs = cv2.resize(np.array(testx[idx]).astype(np.float32), (width, height), interpolation = cv2.INTER_CUBIC)\n",
    "#     obs = obs.reshape(width, height, 1)\n",
    "    \n",
    "#     # Predict an action based on the observation.\n",
    "#     action, _states = inter_model.predict(obs)\n",
    "#     pred_y_test.append(action)\n",
    "\n",
    "\n",
    "#     # Score the prediction.\n",
    "#     if (action.size > 1):\n",
    "#         # print(action)\n",
    "#         count += 1\n",
    "#         pass\n",
    "#     else:\n",
    "#         reward = 1 if action == testy[idx] else 0\n",
    "#         total_rewards += reward\n",
    "\n",
    "# print('Accuracy: {:.2f}%'.format(total_rewards / (len(testy) - count) * 100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "defensive-sheet",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17 \n",
      "\n",
      "Using cpu device\n",
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-63b43d749894>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m17\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0minter_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;31m#@title eval\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-14-d595f6f03889>\u001b[0m in \u001b[0;36mcreate_model\u001b[0;34m(pretrained, save_model, epochs)\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDQN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCustomPolicy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;31m#     model.learn(total_timesteps=300000, log_interval=1)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/dqn/dqn.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    232\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0meval_log_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0meval_log_path\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m             \u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m         )\n\u001b[1;32m    236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    267\u001b[0m                 \u001b[0;31m# do as many gradients steps as steps performed during the rollout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m                 \u001b[0mgradient_steps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient_steps\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient_steps\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mrollout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepisode_timesteps\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 269\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgradient_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    270\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    271\u001b[0m         \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_training_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/dqn/dqn.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, gradient_steps, batch_size)\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mgradient_step\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgradient_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m             \u001b[0;31m# Sample replay buffer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 153\u001b[0;31m             \u001b[0mreplay_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplay_buffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vec_normalize_env\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    154\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/buffers.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, batch_size, env)\u001b[0m\n\u001b[1;32m    232\u001b[0m         \"\"\"\n\u001b[1;32m    233\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize_memory_usage\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m         \u001b[0;31m# Do not sample the element with index `self.pos` as the transitions is invalid\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m         \u001b[0;31m# (we use only one array to store `obs` and `next_obs`)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/buffers.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, batch_size, env)\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0mupper_bound\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer_size\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfull\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m         \u001b[0mbatch_inds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupper_bound\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_samples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_inds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/buffers.py\u001b[0m in \u001b[0;36m_get_samples\u001b[0;34m(self, batch_inds, env)\u001b[0m\n\u001b[1;32m    254\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_normalize_reward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrewards\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbatch_inds\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m         )\n\u001b[0;32m--> 256\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mReplayBufferSamples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_torch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    257\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/vol1/idrl/idrl_env/lib/python3.6/site-packages/stable_baselines3/common/buffers.py\u001b[0m in \u001b[0;36mto_torch\u001b[0;34m(self, array, copy)\u001b[0m\n\u001b[1;32m    126\u001b[0m         \"\"\"\n\u001b[1;32m    127\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    129\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "epochs = [1, 5, 10, 25, 50, 75, 100, 667]\n",
    "# for epoch in epochs[:1]:\n",
    "for epoch in [17]:\n",
    "    print(epoch, '\\n')\n",
    "    inter_model = create_model(pretrained=False, save_model=True, epochs=epoch)\n",
    "    \n",
    "    #@title eval\n",
    "    # Evaluate the model by counting the total rewards attained on the test dataset.\n",
    "    total_rewards = 0\n",
    "    pred_y_test = []\n",
    "    count = 0\n",
    "\n",
    "    for idx in trange(len(testx)):\n",
    "        # Generate an evaluation observation frame.\n",
    "        obs = cv2.resize(np.array(testx[idx]).astype(np.float32), (width, height), interpolation = cv2.INTER_CUBIC)\n",
    "        obs = obs.reshape(width, height, 1)\n",
    "\n",
    "        # Predict an action based on the observation.\n",
    "        action, _states = inter_model.predict(obs)\n",
    "        pred_y_test.append(action)\n",
    "\n",
    "\n",
    "        # Score the prediction.\n",
    "        if (action.size > 1):\n",
    "            # print(action)\n",
    "            count += 1\n",
    "            pass\n",
    "        else:\n",
    "            reward = 1 if action == testy[idx] else 0\n",
    "            total_rewards += reward\n",
    "\n",
    "    print('Accuracy: {:.2f}%'.format(total_rewards / (len(testy) - count) * 100.0))\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "likely-tackle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from glob import glob\n",
    "# import numpy as np\n",
    "\n",
    "# protos = sorted(glob('protos/*'))\n",
    "# critics = sorted(glob('critics/*'))\n",
    "# combined = zip(protos, critics)\n",
    "# combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ordered-robinson",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for p, c in combined:\n",
    "#     m = p.split('_')[-1].split('.')[0]\n",
    "#     npro = np.load(p)\n",
    "#     ncri = np.load(c)\n",
    "#     c = np.concatenate((npro, ncri))\n",
    "#     np.save('combined/combined_{}'.format(m), c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "incomplete-browser",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}