algorithms/8.DDPG.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"DDPG.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true,"authorship_tag":"ABX9TyNVT+UzIZeeDtJ+KOkJ6PXs"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"96483020522b46ba9a3c0cb7f9c0381f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_0bb057c1541149af862682d06e4c7c1c","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_d566e77c690d45568aecf41c40aacd53","IPY_MODEL_b047b706277341759d41c2ade24796ec"]}},"0bb057c1541149af862682d06e4c7c1c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d566e77c690d45568aecf41c40aacd53":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_bf8dc781a95e453bba49ee6de8b37eb7","_dom_classes":[],"description":"100%","_model_name":"FloatProgressModel","bar_style":"success","max":1000,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1000,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_cd0fa01ac5934741bec064d56b72e35b"}},"b047b706277341759d41c2ade24796ec":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_e3661244e052418aa4bcf07be7bd6fb5","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1000/1000 [1:37:16&lt;00:00,  5.84s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_8d674ee11dcd499aa877f8dccd955e59"}},"bf8dc781a95e453bba49ee6de8b37eb7":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"cd0fa01ac5934741bec064d56b72e35b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"e3661244e052418aa4bcf07be7bd6fb5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"8d674ee11dcd499aa877f8dccd955e59":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"code","metadata":{"id":"7GEj8Gptpl9e","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598136246681,"user_tz":-330,"elapsed":4725,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["%%capture\n","!pip install box2d-py"],"execution_count":1,"outputs":[]},{"cell_type":"code","metadata":{"id":"fCgO-BaKpycj","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598136247093,"user_tz":-330,"elapsed":5125,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["import os\n","import cv2\n","import gym\n","import time\n","import collections\n","\n","import numpy as np\n","import torch as T\n","import torch.optim as optim\n","import torch.nn as nn\n","import torch.nn.functional as F\n","import matplotlib.pyplot as plt\n","\n","from tqdm.notebook import tqdm\n","\n","%matplotlib inline"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"d0989dr1p0R2","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":36},"executionInfo":{"status":"ok","timestamp":1598136247098,"user_tz":-330,"elapsed":5119,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}},"outputId":"a889c29d-cfc3-4592-e3bf-f81608062740"},"source":["T.cuda.get_device_name()"],"execution_count":3,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'Tesla T4'"]},"metadata":{"tags":[]},"execution_count":3}]},{"cell_type":"markdown","metadata":{"id":"X1IvliJwp43D","colab_type":"text"},"source":["## **OU Noise**"]},{"cell_type":"code","metadata":{"id":"txmYHTuHp2VG","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598136247101,"user_tz":-330,"elapsed":5114,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["class OUActionNoise:\n","    def __init__(self, mu, sigma=0.15, theta=0.2, dt=1e-2, x0=None):\n","        self.mu = mu\n","        self.sigma = sigma\n","        self.theta = theta\n","        self.dt = dt\n","        self.x0 = x0\n","        self.x_prev = None\n","        self.reset()\n","\n","    def __call__(self):\n","        x = self.x_prev + (self.theta * (self.mu - self.x_prev) * self.dt) +    \\\n","            (self.sigma * np.sqrt(self.dt) * np.random.normal(size=self.mu.shape))\n","        self.x_prev = x\n","        return x\n","\n","    def reset(self):\n","        self.x_prev = self.x0 if self.x0 is not None else np.zeros_like(self.mu)"],"execution_count":4,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"-mdKk7FQOeo4","colab_type":"text"},"source":["## **Replay Buffer**"]},{"cell_type":"code","metadata":{"id":"MM_dfSQKtWHB","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598136247103,"user_tz":-330,"elapsed":5109,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["class ReplayBuffer:\n","    def __init__(self, mem_size, observation_shape, n_actions):\n","        self.mem_size = mem_size\n","        self.mem_counter = 0\n","        # DATA\n","        self.states = np.zeros((mem_size, *observation_shape), dtype=np.float32)\n","        self.actions = np.zeros((mem_size , n_actions), dtype=np.float32)\n","        self.rewards = np.zeros(mem_size, dtype=np.float32)\n","        self.states_ = np.zeros((mem_size, *observation_shape), dtype=np.float32)\n","        self.terminals = np.zeros(mem_size, dtype=bool)\n","\n","    # STORE TRANSITIONS IN BUFFER\n","    def store_transition(self, state, action, reward, state_, done):\n","        index = self.mem_counter % self.mem_size\n","        self.states[index] = state\n","        self.actions[index] = action\n","        self.rewards[index] = reward\n","        self.states_[index] = state_\n","        self.terminals[index] = done # 1 if 'done' else 0\n","        self.mem_counter += 1\n","\n","    # UNIFORMLY SAMPLES 'BUFFER' AND RETURNS A 'BATCH' OF batch_size\n","    def sample_batch(self, batch_size):\n","        max_index = min(self.mem_counter, self.mem_size)\n","        batch_indices = np.random.choice(max_index, batch_size, replace=False)\n","        states = self.states[batch_indices]\n","        actions = self.actions[batch_indices]\n","        rewards = self.rewards[batch_indices]\n","        states_ = self.states_[batch_indices]\n","        terminals = self.terminals[batch_indices]\n","        return (states, actions, rewards, states_, terminals)"],"execution_count":5,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"YWN3WuFUVEB3","colab_type":"text"},"source":["## **Networks**"]},{"cell_type":"code","metadata":{"id":"vkdcl869f1SH","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598136247106,"user_tz":-330,"elapsed":5101,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["class ActorNetwork(nn.Module):\n","    def __init__(self, lr, observation_shape, fc1_size, fc2_size, n_actions, model_name, model_dir):\n","        super().__init__()\n","        self.model_dir = model_dir\n","        self.model_file = os.path.join(self.model_dir, model_name+'_ACTOR')\n","        # ANN\n","        self.fc1 = nn.Linear(observation_shape[0], fc1_size)\n","        self.ln1 = nn.LayerNorm(fc1_size)\n","        self.fc2 = nn.Linear(fc1_size, fc2_size)\n","        self.ln2 = nn.LayerNorm(fc2_size)\n","        self.out = nn.Linear(fc2_size, n_actions)\n","        # FAN INS & WEIGHT INIT\n","        f1 = 1/(np.sqrt(self.fc1.weight.data.size()[0])) \n","        self.fc1.weight.data.uniform_(-f1,f1)\n","        self.fc1.bias.data.uniform_(-f1,f1)\n","        f2 = 1/(np.sqrt(self.fc2.weight.data.size()[0])) \n","        self.fc2.weight.data.uniform_(-f2,f2)\n","        self.fc2.bias.data.uniform_(-f2,f2)\n","        fo = 0.003\n","        self.out.weight.data.uniform_(-fo,fo)\n","        self.out.bias.data.uniform_(-fo,fo)            \n","        # UTILS\n","        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n","        self.optimizer = optim.Adam(self.parameters(), lr=lr)\n","        self.to(self.device)\n","\n","    def forward(self, state):\n","        mu = self.ln1(self.fc1(state))\n","        mu = F.relu(mu)\n","        mu = self.ln2(self.fc2(mu))\n","        mu = F.relu(mu)\n","        mu = self.out(mu)\n","        mu = F.tanh(mu)\n","        return mu #mu = a\n","\n","    def save_model(self):\n","        print(\"[INFO] ACTOR - Saving model\")\n","        checkpoint = {\n","            'model_state_dict': self.state_dict(),\n","            'optimizer_state_dict' : self.optimizer.state_dict()\n","        }\n","        T.save(checkpoint, self.model_file)\n","    \n","    def load_model(self, cpu=False):\n","        print(\"[INFO] ACTOR - Loading model\")\n","        \n","        map_location = T.device('cpu') if (cpu) else None\n","        \n","        checkpoint = T.load(self.model_file, map_location=map_location)\n","        self.load_state_dict(checkpoint['model_state_dict'])\n","        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])"],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"fwSGR9A7VGMx","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598136247108,"user_tz":-330,"elapsed":5096,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["class CriticNetwork(nn.Module):\n","    def __init__(self, lr, observation_shape, fc1_size, fc2_size, n_actions, model_name, model_dir):\n","        super().__init__()\n","        self.model_dir = model_dir\n","        self.model_file = os.path.join(self.model_dir, model_name+'_CRITIC')\n","        # ANN\n","        self.fc1 = nn.Linear(observation_shape[0], fc1_size)\n","        self.ln1 = nn.LayerNorm(fc1_size)\n","        self.fc2 = nn.Linear(fc1_size, fc2_size)\n","        self.ln2 = nn.LayerNorm(fc2_size)\n","        self.fcA = nn.Linear(n_actions, fc2_size)\n","        self.out = nn.Linear(fc2_size, 1)\n","        # FAN INS & WEIGHT INIT \n","        f1 = 1/(np.sqrt(self.fc1.weight.data.size()[0])) \n","        self.fc1.weight.data.uniform_(-f1,f1)\n","        self.fc1.bias.data.uniform_(-f1,f1)\n","        f2 = 1/(np.sqrt(self.fc2.weight.data.size()[0])) \n","        self.fc2.weight.data.uniform_(-f2,f2)\n","        self.fc2.bias.data.uniform_(-f2,f2)\n","        fA = 1/(np.sqrt(self.fcA.weight.data.size()[0]))\n","        self.fcA.weight.data.uniform_(-fA,fA)\n","        self.fcA.bias.data.uniform_(-fA,fA)\n","        fo = 0.003\n","        self.out.weight.data.uniform_(-fo,fo)\n","        self.out.bias.data.uniform_(-fo,fo)\n","        # UTILS\n","        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n","        self.optimizer = optim.Adam(self.parameters(), lr=lr, weight_decay=0.01)\n","        self.to(self.device)\n","\n","    def forward(self, state, action):\n","        s = self.ln1(self.fc1(state))\n","        s = F.relu(s)\n","        s = self.ln2(self.fc2(s))\n","        a = self.fcA(action)\n","        q = T.add(s, a)\n","        q = F.relu(q)\n","        q = self.out(q)\n","        return q\n","\n","    def save_model(self):\n","        print(\"[INFO] CRITIC - Saving model\")\n","        checkpoint = {\n","            'model_state_dict': self.state_dict(),\n","            'optimizer_state_dict' : self.optimizer.state_dict()\n","        }\n","        T.save(checkpoint, self.model_file)\n","    \n","    def load_model(self, cpu=False):\n","        print(\"[INFO] CRITIC - Loading model\")\n","        \n","        map_location = T.device('cpu') if (cpu) else None\n","        \n","        checkpoint = T.load(self.model_file, map_location=map_location)\n","        self.load_state_dict(checkpoint['model_state_dict'])\n","        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])"],"execution_count":7,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"vVJhYxXPpG5a","colab_type":"text"},"source":["## **Agent**"]},{"cell_type":"code","metadata":{"id":"1_2wyD0dpIbG","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598136247111,"user_tz":-330,"elapsed":5088,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["class DDPGAgent:\n","    def __init__(self, observation_shape, n_actions, actor_lr, critic_lr, fc1_size, fc2_size, tau, gamma, \n","                 mem_size, batch_size, algo_name, env_name, model_dir):\n","        self.observation_shape = observation_shape\n","        self.n_actions = n_actions\n","        self.A_LR = actor_lr\n","        self.C_LR = critic_lr\n","        self.TAU = tau\n","        self.GAMMA = gamma\n","        self.noise = OUActionNoise(mu=np.zeros(n_actions))\n","\n","        # MEM PARAMS\n","        self.mem_size = mem_size\n","        self.batch_size = batch_size\n","        self.memory = ReplayBuffer(mem_size, observation_shape, n_actions)\n","\n","        # MODEL PARAMS\n","        self.algo_name = algo_name\n","        self.env_name = env_name\n","        self.model_dir = model_dir\n","        # A1\n","        self.A_STEP = ActorNetwork(actor_lr, observation_shape, fc1_size, fc2_size, n_actions,\n","                              model_name = env_name+'_'+algo_name+'_STEP',\n","                              model_dir = model_dir)\n","        # A2\n","        self.A_TARGET = ActorNetwork(actor_lr, observation_shape, fc1_size, fc2_size, n_actions,\n","                              model_name = env_name+'_'+algo_name+'_TARGET',\n","                              model_dir = model_dir)\n","        # C1\n","        self.C_STEP = CriticNetwork(critic_lr, observation_shape, fc1_size, fc2_size, n_actions,\n","                              model_name = env_name+'_'+algo_name+'_STEP',\n","                              model_dir = model_dir)\n","        # C2\n","        self.C_TARGET = CriticNetwork(critic_lr, observation_shape, fc1_size, fc2_size, n_actions,\n","                              model_name = env_name+'_'+algo_name+'_TARGET',\n","                              model_dir = model_dir)\n","        self.update_target_networks(tau=1) # HARD UPDATE ON INIT\n","    \n","    def get_action(self, observation):\n","        self.A_STEP.eval()\n","        # OBSERVATION TO TENSOR\n","        observation = T.tensor(observation, dtype=T.float32).to(self.A_STEP.device)\n","        state = T.unsqueeze(observation, 0)                                     # T.tensor([array]) = T.unsqueeze(T.tensor(array), 0)\n","        # GET mu = A1(s)\n","        mu = self.A_STEP(state)\n","        mu_PRIME = mu + (T.tensor(self.noise(), dtype=T.float32).to(self.A_STEP.device)) # SHAPE (1,4) + (4)\n","        self.A_STEP.train()\n","        \n","        return mu_PRIME.cpu().detach().numpy()[0]\n","\n","    def learn(self):\n","        if (self.memory.mem_counter < self.batch_size): return # return if insufficient samples present\n","        states, actions, rewards, states_, terminals = self.sample_batch()\n","\n","        # CALC  Q1 = C1(s, a)\n","        q1_preds = self.C_STEP(states, actions)\n","        # CALC  Q2 = y' = R + gamma*(C2(s_, a2_)) where a2_ = A2(s_)\n","        a2_ = self.A_TARGET(states_)\n","        q2_next = self.C_TARGET(states_, a2_)\n","        q2_next[terminals] = 0.0\n","        q2_next = q2_next.view(-1)\n","        q2_targets = rewards + (self.GAMMA * q2_next)\n","        q2_targets = q2_targets.view(self.batch_size, 1)\n","\n","        # CRITIC LOSS\n","        critic_loss = F.mse_loss(q1_preds, q2_targets)\n","        self.C_STEP.optimizer.zero_grad()\n","        critic_loss.backward()\n","        self.C_STEP.optimizer.step()\n","        # ACTOR LOSS\n","        actor_loss = -(self.C_STEP(states, self.A_STEP(states)))\n","        actor_loss = T.mean(actor_loss)\n","        self.A_STEP.optimizer.zero_grad()\n","        actor_loss.backward()\n","        self.A_STEP.optimizer.step()\n","        # UPDATE WEIGHTS OF TARGET NETWORK\n","        self.update_target_networks()\n","\n","    def update_target_networks(self, tau=None):\n","        TAU = self.TAU if tau is None else tau\n","\n","        A1_state_dict =  dict(self.A_STEP.named_parameters())\n","        C1_state_dict =  dict(self.C_STEP.named_parameters())\n","        A2_state_dict =  dict(self.A_TARGET.named_parameters())\n","        C2_state_dict =  dict(self.C_TARGET.named_parameters())\n","        # SOFT UPDATE\n","        for name in A1_state_dict:\n","            A1_state_dict[name] = (TAU *  A1_state_dict[name].clone()) + ((1-TAU)* A2_state_dict[name].clone())\n","        for name in C1_state_dict:\n","            C1_state_dict[name] = (TAU *  C1_state_dict[name].clone()) + ((1-TAU)* C2_state_dict[name].clone())\n","        # UPDATE TARGET WEIGHTS\n","        self.A_TARGET.load_state_dict(A1_state_dict)\n","        self.C_TARGET.load_state_dict(C1_state_dict)\n","\n","        \n","\n","    def store_transition(self, state, action, reward, state_, done):\n","        self.memory.store_transition(state, action, reward, state_, done)\n","\n","    def sample_batch(self):\n","        states, actions, rewards, states_, terminals = self.memory.sample_batch(self.batch_size)\n","        states = T.tensor(states).to(self.A_STEP.device)\n","        actions = T.tensor(actions).to(self.A_STEP.device)\n","        rewards = T.tensor(rewards).to(self.A_STEP.device)\n","        states_ = T.tensor(states_).to(self.A_STEP.device)\n","        terminals = T.tensor(terminals).to(self.A_STEP.device)\n","        return states, actions, rewards, states_, terminals    \n","\n","    def save_models(self):\n","        self.A_STEP.save_model()\n","        self.A_TARGET.save_model()\n","        self.C_STEP.save_model()\n","        self.C_TARGET.save_model()\n","\n","    def load_models(self, cpu=False):\n","        self.A_STEP.load_model(cpu)\n","        self.A_TARGET.load_model(cpu)\n","        self.C_STEP.load_model(cpu)\n","        self.C_TARGET.load_model(cpu)"],"execution_count":8,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"yM1OowM9RCTk","colab_type":"text"},"source":["## **Training**"]},{"cell_type":"code","metadata":{"id":"u1vJrksVRD8w","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598136247112,"user_tz":-330,"elapsed":5081,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["env_name = \"LunarLanderContinuous-v2\"\n","env = gym.make(env_name)\n","\n","N_EPISODES = 1000"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"id":"7JVqH1qGRfiA","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598136251375,"user_tz":-330,"elapsed":9336,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["agent = DDPGAgent(observation_shape=env.observation_space.shape,\n","                  n_actions=env.action_space.shape[0],\n","                  actor_lr=1e-4,\n","                  critic_lr=1e-3,\n","                  fc1_size=400,\n","                  fc2_size=300,\n","                  tau=0.001,\n","                  gamma=0.99,\n","                  mem_size=100000,\n","                  batch_size=64,\n","                  algo_name='DDPG',\n","                  env_name=env_name,\n","                  model_dir='./')"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"2s_9TpQiS-ic","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["96483020522b46ba9a3c0cb7f9c0381f","0bb057c1541149af862682d06e4c7c1c","d566e77c690d45568aecf41c40aacd53","b047b706277341759d41c2ade24796ec","bf8dc781a95e453bba49ee6de8b37eb7","cd0fa01ac5934741bec064d56b72e35b","e3661244e052418aa4bcf07be7bd6fb5","8d674ee11dcd499aa877f8dccd955e59"]},"executionInfo":{"status":"ok","timestamp":1598142087818,"user_tz":-330,"elapsed":5845770,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}},"outputId":"6bb00996-9eb2-4dcf-96ac-dd97752a9f34"},"source":["episode_rewards, episode_lengths, mean_rewards = [],[],[]\n","best_reward = -np.inf\n","\n","for episode_n in tqdm(range(N_EPISODES)):\n","    total_reward, total_moves = 0,0\n","\n","    done = False\n","    agent.noise.reset()\n","    observation = env.reset()\n","\n","    while not done:\n","        action = agent.get_action(observation)\n","        observation_, reward, done, _ = env.step(action)\n","\n","        total_reward += reward\n","        total_moves += 1\n","\n","        # STORE DATA & LEARN\n","        agent.store_transition(observation, action, reward, observation_, done)\n","        agent.learn()\n","\n","        observation = observation_\n","    \n","    episode_rewards.append(total_reward)\n","    episode_lengths.append(total_moves)\n","    \n","    mean_reward = np.mean(episode_rewards[-100:])\n","    mean_rewards.append(mean_reward)\n","    if(mean_reward > best_reward):\n","        agent.save_models()\n","        best_reward = mean_reward\n","    \n","    print(\"ITER: \",episode_n,\"\\tRWD: \",round(total_reward,2),\"\\tM_RWD: \",round(mean_reward,2),\"\\tLEN: \",total_moves)    "],"execution_count":11,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"96483020522b46ba9a3c0cb7f9c0381f","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1614: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n","  warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n"],"name":"stderr"},{"output_type":"stream","text":["[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  0 \tRWD:  -448.57 \tM_RWD:  -448.57 \tLEN:  203\n","ITER:  1 \tRWD:  -1397.77 \tM_RWD:  -923.17 \tLEN:  118\n","ITER:  2 \tRWD:  -339.07 \tM_RWD:  -728.47 \tLEN:  50\n","ITER:  3 \tRWD:  -219.08 \tM_RWD:  -601.12 \tLEN:  62\n","ITER:  4 \tRWD:  -790.99 \tM_RWD:  -639.1 \tLEN:  70\n","ITER:  5 \tRWD:  -1177.87 \tM_RWD:  -728.89 \tLEN:  95\n","ITER:  6 \tRWD:  -786.21 \tM_RWD:  -737.08 \tLEN:  60\n","ITER:  7 \tRWD:  -849.86 \tM_RWD:  -751.18 \tLEN:  78\n","ITER:  8 \tRWD:  -745.25 \tM_RWD:  -750.52 \tLEN:  74\n","ITER:  9 \tRWD:  -405.9 \tM_RWD:  -716.06 \tLEN:  48\n","ITER:  10 \tRWD:  -436.62 \tM_RWD:  -690.65 \tLEN:  59\n","ITER:  11 \tRWD:  -455.99 \tM_RWD:  -671.1 \tLEN:  62\n","ITER:  12 \tRWD:  -803.76 \tM_RWD:  -681.3 \tLEN:  78\n","ITER:  13 \tRWD:  -537.16 \tM_RWD:  -671.01 \tLEN:  69\n","ITER:  14 \tRWD:  -641.22 \tM_RWD:  -669.02 \tLEN:  74\n","ITER:  15 \tRWD:  -622.65 \tM_RWD:  -666.12 \tLEN:  77\n","ITER:  16 \tRWD:  -801.56 \tM_RWD:  -674.09 \tLEN:  86\n","ITER:  17 \tRWD:  -539.32 \tM_RWD:  -666.6 \tLEN:  59\n","ITER:  18 \tRWD:  -670.66 \tM_RWD:  -666.82 \tLEN:  69\n","ITER:  19 \tRWD:  -846.31 \tM_RWD:  -675.79 \tLEN:  81\n","ITER:  20 \tRWD:  -479.24 \tM_RWD:  -666.43 \tLEN:  71\n","ITER:  21 \tRWD:  -614.15 \tM_RWD:  -664.06 \tLEN:  70\n","ITER:  22 \tRWD:  -719.36 \tM_RWD:  -666.46 \tLEN:  79\n","ITER:  23 \tRWD:  -466.06 \tM_RWD:  -658.11 \tLEN:  84\n","ITER:  24 \tRWD:  -848.83 \tM_RWD:  -665.74 \tLEN:  84\n","ITER:  25 \tRWD:  -881.72 \tM_RWD:  -674.05 \tLEN:  91\n","ITER:  26 \tRWD:  -636.4 \tM_RWD:  -672.65 \tLEN:  82\n","ITER:  27 \tRWD:  -500.18 \tM_RWD:  -666.49 \tLEN:  76\n","ITER:  28 \tRWD:  -449.56 \tM_RWD:  -659.01 \tLEN:  75\n","ITER:  29 \tRWD:  -393.52 \tM_RWD:  -650.16 \tLEN:  75\n","ITER:  30 \tRWD:  -665.63 \tM_RWD:  -650.66 \tLEN:  93\n","ITER:  31 \tRWD:  -571.35 \tM_RWD:  -648.18 \tLEN:  64\n","ITER:  32 \tRWD:  -490.13 \tM_RWD:  -643.39 \tLEN:  60\n","ITER:  33 \tRWD:  -629.95 \tM_RWD:  -643.0 \tLEN:  80\n","ITER:  34 \tRWD:  -477.45 \tM_RWD:  -638.27 \tLEN:  70\n","ITER:  35 \tRWD:  -437.25 \tM_RWD:  -632.68 \tLEN:  72\n","ITER:  36 \tRWD:  -319.81 \tM_RWD:  -624.23 \tLEN:  73\n","ITER:  37 \tRWD:  -558.17 \tM_RWD:  -622.49 \tLEN:  102\n","ITER:  38 \tRWD:  -403.79 \tM_RWD:  -616.88 \tLEN:  88\n","ITER:  39 \tRWD:  -289.06 \tM_RWD:  -608.69 \tLEN:  68\n","ITER:  40 \tRWD:  -325.56 \tM_RWD:  -601.78 \tLEN:  85\n","ITER:  41 \tRWD:  -297.71 \tM_RWD:  -594.54 \tLEN:  83\n","ITER:  42 \tRWD:  -609.98 \tM_RWD:  -594.9 \tLEN:  171\n","ITER:  43 \tRWD:  -302.84 \tM_RWD:  -588.26 \tLEN:  94\n","ITER:  44 \tRWD:  -429.25 \tM_RWD:  -584.73 \tLEN:  223\n","ITER:  45 \tRWD:  -108.48 \tM_RWD:  -574.38 \tLEN:  84\n","ITER:  46 \tRWD:  -318.76 \tM_RWD:  -568.94 \tLEN:  285\n","ITER:  47 \tRWD:  -343.15 \tM_RWD:  -564.23 \tLEN:  132\n","ITER:  48 \tRWD:  -19.21 \tM_RWD:  -553.11 \tLEN:  89\n","ITER:  49 \tRWD:  -89.14 \tM_RWD:  -543.83 \tLEN:  153\n","ITER:  50 \tRWD:  -16.72 \tM_RWD:  -533.49 \tLEN:  59\n","ITER:  51 \tRWD:  -317.18 \tM_RWD:  -529.34 \tLEN:  119\n","ITER:  52 \tRWD:  -148.82 \tM_RWD:  -522.16 \tLEN:  51\n","ITER:  53 \tRWD:  -152.73 \tM_RWD:  -515.31 \tLEN:  198\n","ITER:  54 \tRWD:  -325.36 \tM_RWD:  -511.86 \tLEN:  254\n","ITER:  55 \tRWD:  -467.57 \tM_RWD:  -511.07 \tLEN:  89\n","ITER:  56 \tRWD:  -306.8 \tM_RWD:  -507.49 \tLEN:  175\n","ITER:  57 \tRWD:  -155.5 \tM_RWD:  -501.42 \tLEN:  175\n","ITER:  58 \tRWD:  -25.85 \tM_RWD:  -493.36 \tLEN:  200\n","ITER:  59 \tRWD:  -88.86 \tM_RWD:  -486.62 \tLEN:  97\n","ITER:  60 \tRWD:  -216.22 \tM_RWD:  -482.18 \tLEN:  265\n","ITER:  61 \tRWD:  -116.48 \tM_RWD:  -476.28 \tLEN:  126\n","ITER:  62 \tRWD:  -163.28 \tM_RWD:  -471.32 \tLEN:  292\n","ITER:  63 \tRWD:  -263.65 \tM_RWD:  -468.07 \tLEN:  121\n","ITER:  64 \tRWD:  -10.35 \tM_RWD:  -461.03 \tLEN:  84\n","ITER:  65 \tRWD:  -136.62 \tM_RWD:  -456.11 \tLEN:  68\n","ITER:  66 \tRWD:  -33.9 \tM_RWD:  -449.81 \tLEN:  121\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  67 \tRWD:  -243.42 \tM_RWD:  -446.78 \tLEN:  147\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  68 \tRWD:  -56.56 \tM_RWD:  -441.12 \tLEN:  187\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  69 \tRWD:  -32.38 \tM_RWD:  -435.28 \tLEN:  131\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  70 \tRWD:  -314.14 \tM_RWD:  -433.58 \tLEN:  82\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  71 \tRWD:  -168.01 \tM_RWD:  -429.89 \tLEN:  139\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  72 \tRWD:  -70.68 \tM_RWD:  -424.97 \tLEN:  125\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  73 \tRWD:  -225.46 \tM_RWD:  -422.27 \tLEN:  182\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  74 \tRWD:  -209.72 \tM_RWD:  -419.44 \tLEN:  81\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  75 \tRWD:  -166.87 \tM_RWD:  -416.11 \tLEN:  123\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  76 \tRWD:  -172.84 \tM_RWD:  -412.95 \tLEN:  147\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  77 \tRWD:  -91.52 \tM_RWD:  -408.83 \tLEN:  110\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  78 \tRWD:  -59.2 \tM_RWD:  -404.41 \tLEN:  200\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  79 \tRWD:  -95.06 \tM_RWD:  -400.54 \tLEN:  139\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  80 \tRWD:  -233.26 \tM_RWD:  -398.48 \tLEN:  190\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  81 \tRWD:  -108.55 \tM_RWD:  -394.94 \tLEN:  141\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  82 \tRWD:  -143.21 \tM_RWD:  -391.91 \tLEN:  254\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  83 \tRWD:  -75.23 \tM_RWD:  -388.14 \tLEN:  150\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  84 \tRWD:  -138.14 \tM_RWD:  -385.2 \tLEN:  307\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  85 \tRWD:  -78.49 \tM_RWD:  -381.63 \tLEN:  135\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  86 \tRWD:  -84.64 \tM_RWD:  -378.22 \tLEN:  161\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  87 \tRWD:  -176.16 \tM_RWD:  -375.92 \tLEN:  260\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  88 \tRWD:  -214.13 \tM_RWD:  -374.1 \tLEN:  239\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  89 \tRWD:  -101.45 \tM_RWD:  -371.07 \tLEN:  150\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  90 \tRWD:  -215.64 \tM_RWD:  -369.36 \tLEN:  271\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  91 \tRWD:  -138.99 \tM_RWD:  -366.86 \tLEN:  265\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  92 \tRWD:  -260.64 \tM_RWD:  -365.72 \tLEN:  172\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  93 \tRWD:  -101.88 \tM_RWD:  -362.91 \tLEN:  309\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  94 \tRWD:  -321.43 \tM_RWD:  -362.47 \tLEN:  302\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  95 \tRWD:  -221.88 \tM_RWD:  -361.01 \tLEN:  305\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  96 \tRWD:  -177.97 \tM_RWD:  -359.12 \tLEN:  366\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  97 \tRWD:  -162.86 \tM_RWD:  -357.12 \tLEN:  328\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  98 \tRWD:  -270.25 \tM_RWD:  -356.24 \tLEN:  596\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  99 \tRWD:  -159.21 \tM_RWD:  -354.27 \tLEN:  329\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  100 \tRWD:  -106.61 \tM_RWD:  -350.85 \tLEN:  258\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  101 \tRWD:  -127.48 \tM_RWD:  -338.15 \tLEN:  235\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  102 \tRWD:  -213.78 \tM_RWD:  -336.9 \tLEN:  221\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  103 \tRWD:  -164.8 \tM_RWD:  -336.35 \tLEN:  360\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  104 \tRWD:  -69.17 \tM_RWD:  -329.14 \tLEN:  384\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  105 \tRWD:  -70.26 \tM_RWD:  -318.06 \tLEN:  334\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  106 \tRWD:  -145.96 \tM_RWD:  -311.66 \tLEN:  363\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  107 \tRWD:  -319.15 \tM_RWD:  -306.35 \tLEN:  402\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  108 \tRWD:  -121.17 \tM_RWD:  -300.11 \tLEN:  227\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  109 \tRWD:  -174.46 \tM_RWD:  -297.8 \tLEN:  354\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  110 \tRWD:  -211.18 \tM_RWD:  -295.54 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  111 \tRWD:  -175.29 \tM_RWD:  -292.73 \tLEN:  435\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  112 \tRWD:  -455.17 \tM_RWD:  -289.25 \tLEN:  315\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  113 \tRWD:  -155.19 \tM_RWD:  -285.43 \tLEN:  453\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  114 \tRWD:  -199.45 \tM_RWD:  -281.01 \tLEN:  400\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  115 \tRWD:  -495.99 \tM_RWD:  -279.74 \tLEN:  244\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  116 \tRWD:  -142.92 \tM_RWD:  -273.16 \tLEN:  292\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  117 \tRWD:  -144.13 \tM_RWD:  -269.21 \tLEN:  254\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  118 \tRWD:  -360.09 \tM_RWD:  -266.1 \tLEN:  229\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  119 \tRWD:  -177.76 \tM_RWD:  -259.41 \tLEN:  687\n","ITER:  120 \tRWD:  -533.08 \tM_RWD:  -259.95 \tLEN:  319\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  121 \tRWD:  -343.37 \tM_RWD:  -257.24 \tLEN:  422\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  122 \tRWD:  -60.65 \tM_RWD:  -250.66 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  123 \tRWD:  -149.1 \tM_RWD:  -247.49 \tLEN:  473\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  124 \tRWD:  -186.27 \tM_RWD:  -240.86 \tLEN:  252\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  125 \tRWD:  -60.88 \tM_RWD:  -232.65 \tLEN:  791\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  126 \tRWD:  -499.19 \tM_RWD:  -231.28 \tLEN:  248\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  127 \tRWD:  -424.9 \tM_RWD:  -230.53 \tLEN:  184\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  128 \tRWD:  -240.22 \tM_RWD:  -228.44 \tLEN:  520\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  129 \tRWD:  45.74 \tM_RWD:  -224.04 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  130 \tRWD:  -64.79 \tM_RWD:  -218.03 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  131 \tRWD:  -307.45 \tM_RWD:  -215.4 \tLEN:  330\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  132 \tRWD:  -104.84 \tM_RWD:  -211.54 \tLEN:  220\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  133 \tRWD:  -2.25 \tM_RWD:  -205.27 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  134 \tRWD:  -244.87 \tM_RWD:  -202.94 \tLEN:  233\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  135 \tRWD:  -4.41 \tM_RWD:  -198.61 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  136 \tRWD:  -241.29 \tM_RWD:  -197.83 \tLEN:  331\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  137 \tRWD:  -102.96 \tM_RWD:  -193.27 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  138 \tRWD:  -16.68 \tM_RWD:  -189.4 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  139 \tRWD:  -118.36 \tM_RWD:  -187.7 \tLEN:  191\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  140 \tRWD:  -91.29 \tM_RWD:  -185.35 \tLEN:  805\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  141 \tRWD:  -124.66 \tM_RWD:  -183.62 \tLEN:  339\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  142 \tRWD:  -35.48 \tM_RWD:  -177.88 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  143 \tRWD:  -91.12 \tM_RWD:  -175.76 \tLEN:  240\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  144 \tRWD:  -96.96 \tM_RWD:  -172.44 \tLEN:  482\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  145 \tRWD:  -82.31 \tM_RWD:  -172.18 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  146 \tRWD:  -84.36 \tM_RWD:  -169.83 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  147 \tRWD:  -290.23 \tM_RWD:  -169.3 \tLEN:  928\n","ITER:  148 \tRWD:  -70.01 \tM_RWD:  -169.81 \tLEN:  1000\n","ITER:  149 \tRWD:  -48.91 \tM_RWD:  -169.41 \tLEN:  1000\n","ITER:  150 \tRWD:  -27.54 \tM_RWD:  -169.52 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  151 \tRWD:  -182.48 \tM_RWD:  -168.17 \tLEN:  737\n","ITER:  152 \tRWD:  -172.97 \tM_RWD:  -168.41 \tLEN:  602\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  153 \tRWD:  -121.17 \tM_RWD:  -168.1 \tLEN:  644\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  154 \tRWD:  -75.72 \tM_RWD:  -165.6 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  155 \tRWD:  -122.81 \tM_RWD:  -162.15 \tLEN:  572\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  156 \tRWD:  -212.04 \tM_RWD:  -161.2 \tLEN:  671\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  157 \tRWD:  -118.25 \tM_RWD:  -160.83 \tLEN:  555\n","ITER:  158 \tRWD:  -226.69 \tM_RWD:  -162.84 \tLEN:  929\n","ITER:  159 \tRWD:  -136.25 \tM_RWD:  -163.31 \tLEN:  414\n","ITER:  160 \tRWD:  -146.72 \tM_RWD:  -162.62 \tLEN:  609\n","ITER:  161 \tRWD:  -87.09 \tM_RWD:  -162.33 \tLEN:  1000\n","ITER:  162 \tRWD:  -73.63 \tM_RWD:  -161.43 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  163 \tRWD:  -162.2 \tM_RWD:  -160.41 \tLEN:  775\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  164 \tRWD:  -1.79 \tM_RWD:  -160.33 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  165 \tRWD:  -42.32 \tM_RWD:  -159.39 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  166 \tRWD:  -21.71 \tM_RWD:  -159.26 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  167 \tRWD:  -165.91 \tM_RWD:  -158.49 \tLEN:  586\n","ITER:  168 \tRWD:  -86.98 \tM_RWD:  -158.79 \tLEN:  1000\n","ITER:  169 \tRWD:  -58.27 \tM_RWD:  -159.05 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  170 \tRWD:  -51.0 \tM_RWD:  -156.42 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  171 \tRWD:  -24.75 \tM_RWD:  -154.99 \tLEN:  1000\n","ITER:  172 \tRWD:  -119.97 \tM_RWD:  -155.48 \tLEN:  373\n","ITER:  173 \tRWD:  -193.91 \tM_RWD:  -155.17 \tLEN:  842\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  174 \tRWD:  -42.58 \tM_RWD:  -153.49 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  175 \tRWD:  -146.24 \tM_RWD:  -153.29 \tLEN:  477\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  176 \tRWD:  -103.53 \tM_RWD:  -152.59 \tLEN:  355\n","ITER:  177 \tRWD:  -193.92 \tM_RWD:  -153.62 \tLEN:  638\n","ITER:  178 \tRWD:  -13.87 \tM_RWD:  -153.17 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  179 \tRWD:  -36.03 \tM_RWD:  -152.58 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  180 \tRWD:  -144.95 \tM_RWD:  -151.69 \tLEN:  372\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  181 \tRWD:  -98.77 \tM_RWD:  -151.59 \tLEN:  430\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  182 \tRWD:  -116.43 \tM_RWD:  -151.33 \tLEN:  1000\n","ITER:  183 \tRWD:  -98.77 \tM_RWD:  -151.56 \tLEN:  448\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  184 \tRWD:  -39.36 \tM_RWD:  -150.57 \tLEN:  1000\n","ITER:  185 \tRWD:  -157.72 \tM_RWD:  -151.37 \tLEN:  508\n","ITER:  186 \tRWD:  -90.13 \tM_RWD:  -151.42 \tLEN:  296\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  187 \tRWD:  17.17 \tM_RWD:  -149.49 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  188 \tRWD:  -46.47 \tM_RWD:  -147.81 \tLEN:  221\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  189 \tRWD:  -0.6 \tM_RWD:  -146.8 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  190 \tRWD:  20.0 \tM_RWD:  -144.45 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  191 \tRWD:  -27.08 \tM_RWD:  -143.33 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  192 \tRWD:  -124.83 \tM_RWD:  -141.97 \tLEN:  365\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  193 \tRWD:  7.16 \tM_RWD:  -140.88 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  194 \tRWD:  -28.85 \tM_RWD:  -137.95 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  195 \tRWD:  -162.01 \tM_RWD:  -137.35 \tLEN:  668\n","ITER:  196 \tRWD:  -278.98 \tM_RWD:  -138.36 \tLEN:  272\n","ITER:  197 \tRWD:  -125.14 \tM_RWD:  -137.99 \tLEN:  505\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  198 \tRWD:  -108.47 \tM_RWD:  -136.37 \tLEN:  351\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  199 \tRWD:  -123.79 \tM_RWD:  -136.02 \tLEN:  378\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  200 \tRWD:  -65.63 \tM_RWD:  -135.61 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  201 \tRWD:  -39.14 \tM_RWD:  -134.72 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  202 \tRWD:  -181.36 \tM_RWD:  -134.4 \tLEN:  629\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  203 \tRWD:  -18.5 \tM_RWD:  -132.93 \tLEN:  1000\n","ITER:  204 \tRWD:  -423.69 \tM_RWD:  -136.48 \tLEN:  242\n","ITER:  205 \tRWD:  -34.56 \tM_RWD:  -136.12 \tLEN:  1000\n","ITER:  206 \tRWD:  2.15 \tM_RWD:  -134.64 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  207 \tRWD:  -65.06 \tM_RWD:  -132.1 \tLEN:  1000\n","ITER:  208 \tRWD:  -144.46 \tM_RWD:  -132.33 \tLEN:  580\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  209 \tRWD:  -28.78 \tM_RWD:  -130.88 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  210 \tRWD:  -39.84 \tM_RWD:  -129.16 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  211 \tRWD:  -28.64 \tM_RWD:  -127.7 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  212 \tRWD:  12.46 \tM_RWD:  -123.02 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  213 \tRWD:  -19.56 \tM_RWD:  -121.66 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  214 \tRWD:  -25.78 \tM_RWD:  -119.93 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  215 \tRWD:  -24.24 \tM_RWD:  -115.21 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  216 \tRWD:  6.55 \tM_RWD:  -113.72 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  217 \tRWD:  -17.37 \tM_RWD:  -112.45 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  218 \tRWD:  -39.52 \tM_RWD:  -109.24 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  219 \tRWD:  -59.24 \tM_RWD:  -108.06 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  220 \tRWD:  -4.95 \tM_RWD:  -102.78 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  221 \tRWD:  -70.23 \tM_RWD:  -100.04 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  222 \tRWD:  24.76 \tM_RWD:  -99.19 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  223 \tRWD:  -47.82 \tM_RWD:  -98.18 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  224 \tRWD:  12.16 \tM_RWD:  -96.19 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  225 \tRWD:  -29.57 \tM_RWD:  -95.88 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  226 \tRWD:  -61.03 \tM_RWD:  -91.5 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  227 \tRWD:  -29.83 \tM_RWD:  -87.55 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  228 \tRWD:  -25.75 \tM_RWD:  -85.4 \tLEN:  1000\n","ITER:  229 \tRWD:  -37.17 \tM_RWD:  -86.23 \tLEN:  1000\n","ITER:  230 \tRWD:  -15.41 \tM_RWD:  -85.74 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  231 \tRWD:  -51.77 \tM_RWD:  -83.18 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  232 \tRWD:  -49.24 \tM_RWD:  -82.63 \tLEN:  1000\n","ITER:  233 \tRWD:  -51.71 \tM_RWD:  -83.12 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  234 \tRWD:  -19.61 \tM_RWD:  -80.87 \tLEN:  1000\n","ITER:  235 \tRWD:  -10.54 \tM_RWD:  -80.93 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  236 \tRWD:  21.45 \tM_RWD:  -78.3 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  237 \tRWD:  17.97 \tM_RWD:  -77.09 \tLEN:  1000\n","ITER:  238 \tRWD:  -31.23 \tM_RWD:  -77.24 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  239 \tRWD:  -2.24 \tM_RWD:  -76.08 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  240 \tRWD:  -51.75 \tM_RWD:  -75.68 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  241 \tRWD:  -58.58 \tM_RWD:  -75.02 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  242 \tRWD:  -23.73 \tM_RWD:  -74.9 \tLEN:  1000\n","ITER:  243 \tRWD:  -103.92 \tM_RWD:  -75.03 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  244 \tRWD:  -65.58 \tM_RWD:  -74.72 \tLEN:  1000\n","ITER:  245 \tRWD:  -123.62 \tM_RWD:  -75.13 \tLEN:  1000\n","ITER:  246 \tRWD:  -48.89 \tM_RWD:  -74.78 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  247 \tRWD:  -2.97 \tM_RWD:  -71.9 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  248 \tRWD:  -45.75 \tM_RWD:  -71.66 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  249 \tRWD:  -20.71 \tM_RWD:  -71.38 \tLEN:  1000\n","ITER:  250 \tRWD:  -29.22 \tM_RWD:  -71.4 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  251 \tRWD:  6.37 \tM_RWD:  -69.51 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  252 \tRWD:  -45.05 \tM_RWD:  -68.23 \tLEN:  1000\n","ITER:  253 \tRWD:  -154.64 \tM_RWD:  -68.56 \tLEN:  613\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  254 \tRWD:  -13.06 \tM_RWD:  -67.94 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  255 \tRWD:  -54.95 \tM_RWD:  -67.26 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  256 \tRWD:  59.11 \tM_RWD:  -64.55 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  257 \tRWD:  -25.7 \tM_RWD:  -63.62 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  258 \tRWD:  -19.28 \tM_RWD:  -61.55 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  259 \tRWD:  -51.22 \tM_RWD:  -60.7 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  260 \tRWD:  12.43 \tM_RWD:  -59.1 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  261 \tRWD:  -12.19 \tM_RWD:  -58.36 \tLEN:  1000\n","ITER:  262 \tRWD:  -81.18 \tM_RWD:  -58.43 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  263 \tRWD:  -32.95 \tM_RWD:  -57.14 \tLEN:  1000\n","ITER:  264 \tRWD:  -43.55 \tM_RWD:  -57.56 \tLEN:  1000\n","ITER:  265 \tRWD:  -230.9 \tM_RWD:  -59.44 \tLEN:  332\n","ITER:  266 \tRWD:  -28.52 \tM_RWD:  -59.51 \tLEN:  1000\n","ITER:  267 \tRWD:  -38.03 \tM_RWD:  -58.23 \tLEN:  1000\n","ITER:  268 \tRWD:  -95.2 \tM_RWD:  -58.31 \tLEN:  244\n","ITER:  269 \tRWD:  -35.15 \tM_RWD:  -58.08 \tLEN:  1000\n","ITER:  270 \tRWD:  -33.77 \tM_RWD:  -57.91 \tLEN:  1000\n","ITER:  271 \tRWD:  32.71 \tM_RWD:  -57.33 \tLEN:  92\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  272 \tRWD:  -0.83 \tM_RWD:  -56.14 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  273 \tRWD:  -72.42 \tM_RWD:  -54.93 \tLEN:  1000\n","ITER:  274 \tRWD:  -119.62 \tM_RWD:  -55.7 \tLEN:  945\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  275 \tRWD:  -11.82 \tM_RWD:  -54.35 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  276 \tRWD:  -4.26 \tM_RWD:  -53.36 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  277 \tRWD:  -22.57 \tM_RWD:  -51.65 \tLEN:  1000\n","ITER:  278 \tRWD:  -60.71 \tM_RWD:  -52.12 \tLEN:  1000\n","ITER:  279 \tRWD:  -15.53 \tM_RWD:  -51.91 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  280 \tRWD:  -54.39 \tM_RWD:  -51.01 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  281 \tRWD:  -32.2 \tM_RWD:  -50.34 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  282 \tRWD:  13.14 \tM_RWD:  -49.04 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  283 \tRWD:  -32.98 \tM_RWD:  -48.39 \tLEN:  1000\n","ITER:  284 \tRWD:  -112.0 \tM_RWD:  -49.11 \tLEN:  55\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  285 \tRWD:  -60.96 \tM_RWD:  -48.15 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  286 \tRWD:  0.39 \tM_RWD:  -47.24 \tLEN:  1000\n","ITER:  287 \tRWD:  9.57 \tM_RWD:  -47.32 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  288 \tRWD:  -22.1 \tM_RWD:  -47.07 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  289 \tRWD:  137.73 \tM_RWD:  -45.69 \tLEN:  784\n","ITER:  290 \tRWD:  -83.51 \tM_RWD:  -46.72 \tLEN:  1000\n","ITER:  291 \tRWD:  -16.33 \tM_RWD:  -46.62 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  292 \tRWD:  38.98 \tM_RWD:  -44.98 \tLEN:  1000\n","ITER:  293 \tRWD:  -39.45 \tM_RWD:  -45.45 \tLEN:  1000\n","ITER:  294 \tRWD:  -112.87 \tM_RWD:  -46.29 \tLEN:  85\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  295 \tRWD:  198.53 \tM_RWD:  -42.68 \tLEN:  347\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  296 \tRWD:  -217.14 \tM_RWD:  -42.06 \tLEN:  70\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  297 \tRWD:  -90.76 \tM_RWD:  -41.72 \tLEN:  135\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  298 \tRWD:  -35.58 \tM_RWD:  -40.99 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  299 \tRWD:  121.34 \tM_RWD:  -38.54 \tLEN:  912\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  300 \tRWD:  -41.78 \tM_RWD:  -38.3 \tLEN:  408\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  301 \tRWD:  180.68 \tM_RWD:  -36.1 \tLEN:  344\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  302 \tRWD:  42.93 \tM_RWD:  -33.86 \tLEN:  843\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  303 \tRWD:  66.69 \tM_RWD:  -33.01 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  304 \tRWD:  -83.09 \tM_RWD:  -29.6 \tLEN:  1000\n","ITER:  305 \tRWD:  -34.79 \tM_RWD:  -29.6 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  306 \tRWD:  29.2 \tM_RWD:  -29.33 \tLEN:  203\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  307 \tRWD:  -43.97 \tM_RWD:  -29.12 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  308 \tRWD:  148.29 \tM_RWD:  -26.19 \tLEN:  322\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  309 \tRWD:  161.41 \tM_RWD:  -24.29 \tLEN:  946\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  310 \tRWD:  97.08 \tM_RWD:  -22.92 \tLEN:  766\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  311 \tRWD:  42.91 \tM_RWD:  -22.21 \tLEN:  172\n","ITER:  312 \tRWD:  -57.16 \tM_RWD:  -22.9 \tLEN:  88\n","ITER:  313 \tRWD:  -100.86 \tM_RWD:  -23.72 \tLEN:  263\n","ITER:  314 \tRWD:  -29.88 \tM_RWD:  -23.76 \tLEN:  238\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  315 \tRWD:  211.97 \tM_RWD:  -21.39 \tLEN:  210\n","ITER:  316 \tRWD:  -66.55 \tM_RWD:  -22.13 \tLEN:  263\n","ITER:  317 \tRWD:  -102.42 \tM_RWD:  -22.98 \tLEN:  306\n","ITER:  318 \tRWD:  -7.42 \tM_RWD:  -22.66 \tLEN:  158\n","ITER:  319 \tRWD:  -66.21 \tM_RWD:  -22.73 \tLEN:  229\n","ITER:  320 \tRWD:  -56.7 \tM_RWD:  -23.24 \tLEN:  212\n","ITER:  321 \tRWD:  -45.88 \tM_RWD:  -23.0 \tLEN:  279\n","ITER:  322 \tRWD:  -63.59 \tM_RWD:  -23.88 \tLEN:  204\n","ITER:  323 \tRWD:  -61.26 \tM_RWD:  -24.02 \tLEN:  233\n","ITER:  324 \tRWD:  -95.95 \tM_RWD:  -25.1 \tLEN:  1000\n","ITER:  325 \tRWD:  -158.35 \tM_RWD:  -26.39 \tLEN:  265\n","ITER:  326 \tRWD:  165.55 \tM_RWD:  -24.12 \tLEN:  367\n","ITER:  327 \tRWD:  -55.65 \tM_RWD:  -24.38 \tLEN:  168\n","ITER:  328 \tRWD:  -17.82 \tM_RWD:  -24.3 \tLEN:  1000\n","ITER:  329 \tRWD:  -297.05 \tM_RWD:  -26.9 \tLEN:  122\n","ITER:  330 \tRWD:  -15.03 \tM_RWD:  -26.89 \tLEN:  300\n","ITER:  331 \tRWD:  -37.06 \tM_RWD:  -26.75 \tLEN:  767\n","ITER:  332 \tRWD:  -138.21 \tM_RWD:  -27.64 \tLEN:  186\n","ITER:  333 \tRWD:  -44.67 \tM_RWD:  -27.57 \tLEN:  1000\n","ITER:  334 \tRWD:  -362.62 \tM_RWD:  -31.0 \tLEN:  130\n","ITER:  335 \tRWD:  -43.73 \tM_RWD:  -31.33 \tLEN:  1000\n","ITER:  336 \tRWD:  -22.59 \tM_RWD:  -31.77 \tLEN:  1000\n","ITER:  337 \tRWD:  -20.03 \tM_RWD:  -32.15 \tLEN:  1000\n","ITER:  338 \tRWD:  -48.89 \tM_RWD:  -32.33 \tLEN:  1000\n","ITER:  339 \tRWD:  -85.8 \tM_RWD:  -33.16 \tLEN:  1000\n","ITER:  340 \tRWD:  -198.73 \tM_RWD:  -34.63 \tLEN:  1000\n","ITER:  341 \tRWD:  -48.24 \tM_RWD:  -34.53 \tLEN:  1000\n","ITER:  342 \tRWD:  -46.02 \tM_RWD:  -34.75 \tLEN:  1000\n","ITER:  343 \tRWD:  -198.3 \tM_RWD:  -35.69 \tLEN:  112\n","ITER:  344 \tRWD:  -161.28 \tM_RWD:  -36.65 \tLEN:  137\n","ITER:  345 \tRWD:  -51.19 \tM_RWD:  -35.93 \tLEN:  1000\n","ITER:  346 \tRWD:  -24.12 \tM_RWD:  -35.68 \tLEN:  1000\n","ITER:  347 \tRWD:  -41.66 \tM_RWD:  -36.07 \tLEN:  1000\n","ITER:  348 \tRWD:  -279.07 \tM_RWD:  -38.4 \tLEN:  126\n","ITER:  349 \tRWD:  -523.91 \tM_RWD:  -43.43 \tLEN:  89\n","ITER:  350 \tRWD:  -23.83 \tM_RWD:  -43.38 \tLEN:  1000\n","ITER:  351 \tRWD:  -219.01 \tM_RWD:  -45.63 \tLEN:  380\n","ITER:  352 \tRWD:  -372.85 \tM_RWD:  -48.91 \tLEN:  122\n","ITER:  353 \tRWD:  -317.58 \tM_RWD:  -50.54 \tLEN:  108\n","ITER:  354 \tRWD:  272.5 \tM_RWD:  -47.68 \tLEN:  337\n","ITER:  355 \tRWD:  -3.36 \tM_RWD:  -47.17 \tLEN:  1000\n","ITER:  356 \tRWD:  -63.44 \tM_RWD:  -48.39 \tLEN:  1000\n","ITER:  357 \tRWD:  -146.07 \tM_RWD:  -49.6 \tLEN:  1000\n","ITER:  358 \tRWD:  -107.19 \tM_RWD:  -50.47 \tLEN:  1000\n","ITER:  359 \tRWD:  -98.34 \tM_RWD:  -50.95 \tLEN:  1000\n","ITER:  360 \tRWD:  155.61 \tM_RWD:  -49.51 \tLEN:  814\n","ITER:  361 \tRWD:  -52.04 \tM_RWD:  -49.91 \tLEN:  1000\n","ITER:  362 \tRWD:  -254.7 \tM_RWD:  -51.65 \tLEN:  822\n","ITER:  363 \tRWD:  -252.84 \tM_RWD:  -53.85 \tLEN:  416\n","ITER:  364 \tRWD:  -263.58 \tM_RWD:  -56.05 \tLEN:  471\n","ITER:  365 \tRWD:  40.96 \tM_RWD:  -53.33 \tLEN:  85\n","ITER:  366 \tRWD:  -340.56 \tM_RWD:  -56.45 \tLEN:  304\n","ITER:  367 \tRWD:  -40.73 \tM_RWD:  -56.48 \tLEN:  1000\n","ITER:  368 \tRWD:  -38.8 \tM_RWD:  -55.91 \tLEN:  492\n","ITER:  369 \tRWD:  -246.97 \tM_RWD:  -58.03 \tLEN:  397\n","ITER:  370 \tRWD:  -175.6 \tM_RWD:  -59.45 \tLEN:  737\n","ITER:  371 \tRWD:  14.54 \tM_RWD:  -59.63 \tLEN:  1000\n","ITER:  372 \tRWD:  -488.58 \tM_RWD:  -64.51 \tLEN:  802\n","ITER:  373 \tRWD:  146.02 \tM_RWD:  -62.32 \tLEN:  876\n","ITER:  374 \tRWD:  156.36 \tM_RWD:  -59.56 \tLEN:  754\n","ITER:  375 \tRWD:  6.85 \tM_RWD:  -59.38 \tLEN:  1000\n","ITER:  376 \tRWD:  -273.31 \tM_RWD:  -62.07 \tLEN:  483\n","ITER:  377 \tRWD:  174.48 \tM_RWD:  -60.1 \tLEN:  928\n","ITER:  378 \tRWD:  134.55 \tM_RWD:  -58.14 \tLEN:  981\n","ITER:  379 \tRWD:  157.63 \tM_RWD:  -56.41 \tLEN:  964\n","ITER:  380 \tRWD:  2.33 \tM_RWD:  -55.85 \tLEN:  1000\n","ITER:  381 \tRWD:  11.28 \tM_RWD:  -55.41 \tLEN:  304\n","ITER:  382 \tRWD:  -402.68 \tM_RWD:  -59.57 \tLEN:  941\n","ITER:  383 \tRWD:  4.25 \tM_RWD:  -59.2 \tLEN:  1000\n","ITER:  384 \tRWD:  12.85 \tM_RWD:  -57.95 \tLEN:  1000\n","ITER:  385 \tRWD:  -14.99 \tM_RWD:  -57.49 \tLEN:  1000\n","ITER:  386 \tRWD:  -29.01 \tM_RWD:  -57.78 \tLEN:  1000\n","ITER:  387 \tRWD:  -20.28 \tM_RWD:  -58.08 \tLEN:  1000\n","ITER:  388 \tRWD:  -160.63 \tM_RWD:  -59.47 \tLEN:  411\n","ITER:  389 \tRWD:  -19.44 \tM_RWD:  -61.04 \tLEN:  1000\n","ITER:  390 \tRWD:  5.5 \tM_RWD:  -60.15 \tLEN:  1000\n","ITER:  391 \tRWD:  -9.83 \tM_RWD:  -60.08 \tLEN:  1000\n","ITER:  392 \tRWD:  24.32 \tM_RWD:  -60.23 \tLEN:  1000\n","ITER:  393 \tRWD:  -46.56 \tM_RWD:  -60.3 \tLEN:  1000\n","ITER:  394 \tRWD:  -25.62 \tM_RWD:  -59.43 \tLEN:  1000\n","ITER:  395 \tRWD:  -26.77 \tM_RWD:  -61.68 \tLEN:  1000\n","ITER:  396 \tRWD:  -14.65 \tM_RWD:  -59.66 \tLEN:  1000\n","ITER:  397 \tRWD:  14.15 \tM_RWD:  -58.61 \tLEN:  1000\n","ITER:  398 \tRWD:  -38.82 \tM_RWD:  -58.64 \tLEN:  1000\n","ITER:  399 \tRWD:  -8.25 \tM_RWD:  -59.94 \tLEN:  1000\n","ITER:  400 \tRWD:  -717.71 \tM_RWD:  -66.69 \tLEN:  242\n","ITER:  401 \tRWD:  -27.02 \tM_RWD:  -68.77 \tLEN:  1000\n","ITER:  402 \tRWD:  -34.34 \tM_RWD:  -69.54 \tLEN:  1000\n","ITER:  403 \tRWD:  -303.98 \tM_RWD:  -73.25 \tLEN:  505\n","ITER:  404 \tRWD:  -15.93 \tM_RWD:  -72.58 \tLEN:  1000\n","ITER:  405 \tRWD:  68.62 \tM_RWD:  -71.55 \tLEN:  1000\n","ITER:  406 \tRWD:  -9.79 \tM_RWD:  -71.94 \tLEN:  1000\n","ITER:  407 \tRWD:  -39.21 \tM_RWD:  -71.89 \tLEN:  1000\n","ITER:  408 \tRWD:  -55.48 \tM_RWD:  -73.93 \tLEN:  1000\n","ITER:  409 \tRWD:  50.12 \tM_RWD:  -75.04 \tLEN:  1000\n","ITER:  410 \tRWD:  164.74 \tM_RWD:  -74.36 \tLEN:  690\n","ITER:  411 \tRWD:  136.57 \tM_RWD:  -73.43 \tLEN:  836\n","ITER:  412 \tRWD:  9.27 \tM_RWD:  -72.76 \tLEN:  1000\n","ITER:  413 \tRWD:  -19.93 \tM_RWD:  -71.95 \tLEN:  1000\n","ITER:  414 \tRWD:  170.32 \tM_RWD:  -69.95 \tLEN:  990\n","ITER:  415 \tRWD:  35.1 \tM_RWD:  -71.72 \tLEN:  1000\n","ITER:  416 \tRWD:  -30.36 \tM_RWD:  -71.36 \tLEN:  1000\n","ITER:  417 \tRWD:  209.26 \tM_RWD:  -68.24 \tLEN:  684\n","ITER:  418 \tRWD:  163.16 \tM_RWD:  -66.53 \tLEN:  378\n","ITER:  419 \tRWD:  38.86 \tM_RWD:  -65.48 \tLEN:  1000\n","ITER:  420 \tRWD:  81.06 \tM_RWD:  -64.11 \tLEN:  842\n","ITER:  421 \tRWD:  117.06 \tM_RWD:  -62.48 \tLEN:  995\n","ITER:  422 \tRWD:  255.35 \tM_RWD:  -59.29 \tLEN:  301\n","ITER:  423 \tRWD:  255.22 \tM_RWD:  -56.12 \tLEN:  332\n","ITER:  424 \tRWD:  146.43 \tM_RWD:  -53.7 \tLEN:  1000\n","ITER:  425 \tRWD:  180.55 \tM_RWD:  -50.31 \tLEN:  444\n","ITER:  426 \tRWD:  155.83 \tM_RWD:  -50.41 \tLEN:  814\n","ITER:  427 \tRWD:  -3.87 \tM_RWD:  -49.89 \tLEN:  193\n","ITER:  428 \tRWD:  276.35 \tM_RWD:  -46.95 \tLEN:  521\n","ITER:  429 \tRWD:  -395.28 \tM_RWD:  -47.93 \tLEN:  311\n","ITER:  430 \tRWD:  201.18 \tM_RWD:  -45.77 \tLEN:  769\n","ITER:  431 \tRWD:  283.44 \tM_RWD:  -42.56 \tLEN:  312\n","ITER:  432 \tRWD:  283.09 \tM_RWD:  -38.35 \tLEN:  157\n","ITER:  433 \tRWD:  181.53 \tM_RWD:  -36.09 \tLEN:  334\n","ITER:  434 \tRWD:  250.07 \tM_RWD:  -29.96 \tLEN:  286\n","ITER:  435 \tRWD:  211.22 \tM_RWD:  -27.41 \tLEN:  626\n","ITER:  436 \tRWD:  207.28 \tM_RWD:  -25.11 \tLEN:  372\n","ITER:  437 \tRWD:  164.82 \tM_RWD:  -23.26 \tLEN:  553\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  438 \tRWD:  256.95 \tM_RWD:  -20.21 \tLEN:  462\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  439 \tRWD:  256.88 \tM_RWD:  -16.78 \tLEN:  211\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  440 \tRWD:  242.72 \tM_RWD:  -12.36 \tLEN:  534\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  441 \tRWD:  144.52 \tM_RWD:  -10.44 \tLEN:  706\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  442 \tRWD:  180.99 \tM_RWD:  -8.17 \tLEN:  659\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  443 \tRWD:  251.81 \tM_RWD:  -3.66 \tLEN:  556\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  444 \tRWD:  260.01 \tM_RWD:  0.55 \tLEN:  178\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  445 \tRWD:  216.28 \tM_RWD:  3.22 \tLEN:  256\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  446 \tRWD:  34.59 \tM_RWD:  3.81 \tLEN:  172\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  447 \tRWD:  301.72 \tM_RWD:  7.24 \tLEN:  291\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  448 \tRWD:  29.29 \tM_RWD:  10.33 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  449 \tRWD:  234.36 \tM_RWD:  17.91 \tLEN:  230\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  450 \tRWD:  14.71 \tM_RWD:  18.3 \tLEN:  107\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  451 \tRWD:  209.26 \tM_RWD:  22.58 \tLEN:  381\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  452 \tRWD:  195.45 \tM_RWD:  28.26 \tLEN:  604\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  453 \tRWD:  239.41 \tM_RWD:  33.83 \tLEN:  355\n","ITER:  454 \tRWD:  238.09 \tM_RWD:  33.49 \tLEN:  399\n","ITER:  455 \tRWD:  -54.96 \tM_RWD:  32.97 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  456 \tRWD:  175.71 \tM_RWD:  35.36 \tLEN:  700\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  457 \tRWD:  19.98 \tM_RWD:  37.02 \tLEN:  115\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  458 \tRWD:  235.93 \tM_RWD:  40.45 \tLEN:  213\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  459 \tRWD:  41.54 \tM_RWD:  41.85 \tLEN:  117\n","ITER:  460 \tRWD:  -4.31 \tM_RWD:  40.25 \tLEN:  88\n","ITER:  461 \tRWD:  12.55 \tM_RWD:  40.9 \tLEN:  134\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  462 \tRWD:  216.08 \tM_RWD:  45.61 \tLEN:  186\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  463 \tRWD:  -212.28 \tM_RWD:  46.01 \tLEN:  112\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  464 \tRWD:  191.84 \tM_RWD:  50.57 \tLEN:  470\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  465 \tRWD:  258.4 \tM_RWD:  52.74 \tLEN:  196\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  466 \tRWD:  37.98 \tM_RWD:  56.53 \tLEN:  123\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  467 \tRWD:  268.69 \tM_RWD:  59.62 \tLEN:  204\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  468 \tRWD:  14.77 \tM_RWD:  60.16 \tLEN:  91\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  469 \tRWD:  265.38 \tM_RWD:  65.28 \tLEN:  369\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  470 \tRWD:  262.51 \tM_RWD:  69.66 \tLEN:  195\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  471 \tRWD:  236.6 \tM_RWD:  71.88 \tLEN:  702\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  472 \tRWD:  262.32 \tM_RWD:  79.39 \tLEN:  204\n","ITER:  473 \tRWD:  -31.51 \tM_RWD:  77.62 \tLEN:  83\n","ITER:  474 \tRWD:  274.4 \tM_RWD:  78.8 \tLEN:  231\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  475 \tRWD:  232.37 \tM_RWD:  81.05 \tLEN:  311\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  476 \tRWD:  238.96 \tM_RWD:  86.17 \tLEN:  323\n","ITER:  477 \tRWD:  41.99 \tM_RWD:  84.85 \tLEN:  130\n","ITER:  478 \tRWD:  258.41 \tM_RWD:  86.09 \tLEN:  301\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  479 \tRWD:  263.37 \tM_RWD:  87.15 \tLEN:  199\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  480 \tRWD:  268.12 \tM_RWD:  89.8 \tLEN:  412\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  481 \tRWD:  234.53 \tM_RWD:  92.04 \tLEN:  293\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  482 \tRWD:  30.62 \tM_RWD:  96.37 \tLEN:  1000\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  483 \tRWD:  295.54 \tM_RWD:  99.28 \tLEN:  222\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  484 \tRWD:  275.22 \tM_RWD:  101.91 \tLEN:  175\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  485 \tRWD:  250.93 \tM_RWD:  104.56 \tLEN:  258\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  486 \tRWD:  255.75 \tM_RWD:  107.41 \tLEN:  456\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  487 \tRWD:  251.68 \tM_RWD:  110.13 \tLEN:  229\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  488 \tRWD:  46.63 \tM_RWD:  112.2 \tLEN:  145\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  489 \tRWD:  238.23 \tM_RWD:  114.78 \tLEN:  408\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  490 \tRWD:  257.56 \tM_RWD:  117.3 \tLEN:  189\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  491 \tRWD:  229.6 \tM_RWD:  119.7 \tLEN:  300\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  492 \tRWD:  209.39 \tM_RWD:  121.55 \tLEN:  265\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  493 \tRWD:  192.14 \tM_RWD:  123.93 \tLEN:  308\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  494 \tRWD:  44.41 \tM_RWD:  124.63 \tLEN:  132\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  495 \tRWD:  279.22 \tM_RWD:  127.69 \tLEN:  322\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  496 \tRWD:  259.98 \tM_RWD:  130.44 \tLEN:  235\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  497 \tRWD:  240.33 \tM_RWD:  132.7 \tLEN:  201\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  498 \tRWD:  252.79 \tM_RWD:  135.62 \tLEN:  254\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  499 \tRWD:  258.4 \tM_RWD:  138.28 \tLEN:  193\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  500 \tRWD:  33.45 \tM_RWD:  145.8 \tLEN:  210\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  501 \tRWD:  271.76 \tM_RWD:  148.78 \tLEN:  193\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  502 \tRWD:  250.67 \tM_RWD:  151.63 \tLEN:  236\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  503 \tRWD:  270.17 \tM_RWD:  157.38 \tLEN:  201\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  504 \tRWD:  244.04 \tM_RWD:  159.98 \tLEN:  436\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  505 \tRWD:  273.65 \tM_RWD:  162.03 \tLEN:  215\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  506 \tRWD:  285.49 \tM_RWD:  164.98 \tLEN:  248\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  507 \tRWD:  252.36 \tM_RWD:  167.89 \tLEN:  340\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  508 \tRWD:  226.23 \tM_RWD:  170.71 \tLEN:  183\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  509 \tRWD:  281.0 \tM_RWD:  173.02 \tLEN:  366\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  510 \tRWD:  278.9 \tM_RWD:  174.16 \tLEN:  298\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  511 \tRWD:  279.96 \tM_RWD:  175.6 \tLEN:  337\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  512 \tRWD:  241.51 \tM_RWD:  177.92 \tLEN:  197\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  513 \tRWD:  100.0 \tM_RWD:  179.12 \tLEN:  720\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  514 \tRWD:  275.34 \tM_RWD:  180.17 \tLEN:  223\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  515 \tRWD:  277.52 \tM_RWD:  182.59 \tLEN:  253\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  516 \tRWD:  272.84 \tM_RWD:  185.62 \tLEN:  375\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  517 \tRWD:  281.0 \tM_RWD:  186.34 \tLEN:  305\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  518 \tRWD:  292.54 \tM_RWD:  187.63 \tLEN:  352\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  519 \tRWD:  267.77 \tM_RWD:  189.92 \tLEN:  474\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  520 \tRWD:  247.81 \tM_RWD:  191.59 \tLEN:  214\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  521 \tRWD:  264.39 \tM_RWD:  193.06 \tLEN:  290\n","ITER:  522 \tRWD:  -215.55 \tM_RWD:  188.36 \tLEN:  269\n","ITER:  523 \tRWD:  267.8 \tM_RWD:  188.48 \tLEN:  391\n","ITER:  524 \tRWD:  68.71 \tM_RWD:  187.7 \tLEN:  111\n","ITER:  525 \tRWD:  -408.09 \tM_RWD:  181.82 \tLEN:  392\n","ITER:  526 \tRWD:  271.07 \tM_RWD:  182.97 \tLEN:  325\n","ITER:  527 \tRWD:  -561.98 \tM_RWD:  177.39 \tLEN:  357\n","ITER:  528 \tRWD:  -20.15 \tM_RWD:  174.42 \tLEN:  105\n","ITER:  529 \tRWD:  -345.04 \tM_RWD:  174.93 \tLEN:  384\n","ITER:  530 \tRWD:  -9.18 \tM_RWD:  172.82 \tLEN:  183\n","ITER:  531 \tRWD:  -677.63 \tM_RWD:  163.21 \tLEN:  233\n","ITER:  532 \tRWD:  -424.56 \tM_RWD:  156.14 \tLEN:  181\n","ITER:  533 \tRWD:  260.38 \tM_RWD:  156.92 \tLEN:  305\n","ITER:  534 \tRWD:  -559.34 \tM_RWD:  148.83 \tLEN:  350\n","ITER:  535 \tRWD:  246.32 \tM_RWD:  149.18 \tLEN:  401\n","ITER:  536 \tRWD:  255.95 \tM_RWD:  149.67 \tLEN:  488\n","ITER:  537 \tRWD:  205.55 \tM_RWD:  150.07 \tLEN:  338\n","ITER:  538 \tRWD:  -241.99 \tM_RWD:  145.09 \tLEN:  172\n","ITER:  539 \tRWD:  220.3 \tM_RWD:  144.72 \tLEN:  298\n","ITER:  540 \tRWD:  240.34 \tM_RWD:  144.7 \tLEN:  340\n","ITER:  541 \tRWD:  206.63 \tM_RWD:  145.32 \tLEN:  393\n","ITER:  542 \tRWD:  -170.58 \tM_RWD:  141.8 \tLEN:  241\n","ITER:  543 \tRWD:  222.07 \tM_RWD:  141.5 \tLEN:  246\n","ITER:  544 \tRWD:  -148.16 \tM_RWD:  137.42 \tLEN:  219\n","ITER:  545 \tRWD:  262.73 \tM_RWD:  137.89 \tLEN:  272\n","ITER:  546 \tRWD:  283.23 \tM_RWD:  140.37 \tLEN:  286\n","ITER:  547 \tRWD:  183.93 \tM_RWD:  139.19 \tLEN:  214\n","ITER:  548 \tRWD:  238.97 \tM_RWD:  141.29 \tLEN:  179\n","ITER:  549 \tRWD:  260.22 \tM_RWD:  141.55 \tLEN:  331\n","ITER:  550 \tRWD:  260.16 \tM_RWD:  144.0 \tLEN:  312\n","ITER:  551 \tRWD:  262.06 \tM_RWD:  144.53 \tLEN:  277\n","ITER:  552 \tRWD:  -196.05 \tM_RWD:  140.62 \tLEN:  229\n","ITER:  553 \tRWD:  -193.3 \tM_RWD:  136.29 \tLEN:  291\n","ITER:  554 \tRWD:  -208.64 \tM_RWD:  131.82 \tLEN:  212\n","ITER:  555 \tRWD:  269.04 \tM_RWD:  135.06 \tLEN:  416\n","ITER:  556 \tRWD:  10.64 \tM_RWD:  133.41 \tLEN:  158\n","ITER:  557 \tRWD:  233.09 \tM_RWD:  135.54 \tLEN:  379\n","ITER:  558 \tRWD:  269.25 \tM_RWD:  135.88 \tLEN:  242\n","ITER:  559 \tRWD:  246.29 \tM_RWD:  137.92 \tLEN:  319\n","ITER:  560 \tRWD:  -121.37 \tM_RWD:  136.75 \tLEN:  184\n","ITER:  561 \tRWD:  224.23 \tM_RWD:  138.87 \tLEN:  289\n","ITER:  562 \tRWD:  287.45 \tM_RWD:  139.58 \tLEN:  274\n","ITER:  563 \tRWD:  233.68 \tM_RWD:  144.04 \tLEN:  414\n","ITER:  564 \tRWD:  226.82 \tM_RWD:  144.39 \tLEN:  707\n","ITER:  565 \tRWD:  1.82 \tM_RWD:  141.83 \tLEN:  347\n","ITER:  566 \tRWD:  -395.99 \tM_RWD:  137.49 \tLEN:  458\n","ITER:  567 \tRWD:  259.9 \tM_RWD:  137.4 \tLEN:  307\n","ITER:  568 \tRWD:  168.94 \tM_RWD:  138.94 \tLEN:  378\n","ITER:  569 \tRWD:  196.56 \tM_RWD:  138.25 \tLEN:  388\n","ITER:  570 \tRWD:  211.14 \tM_RWD:  137.74 \tLEN:  371\n","ITER:  571 \tRWD:  -77.55 \tM_RWD:  134.6 \tLEN:  167\n","ITER:  572 \tRWD:  165.08 \tM_RWD:  133.63 \tLEN:  515\n","ITER:  573 \tRWD:  -55.85 \tM_RWD:  133.38 \tLEN:  240\n","ITER:  574 \tRWD:  254.21 \tM_RWD:  133.18 \tLEN:  336\n","ITER:  575 \tRWD:  -0.85 \tM_RWD:  130.85 \tLEN:  361\n","ITER:  576 \tRWD:  41.19 \tM_RWD:  128.87 \tLEN:  234\n","ITER:  577 \tRWD:  -87.42 \tM_RWD:  127.58 \tLEN:  268\n","ITER:  578 \tRWD:  238.24 \tM_RWD:  127.38 \tLEN:  364\n","ITER:  579 \tRWD:  -82.44 \tM_RWD:  123.92 \tLEN:  218\n","ITER:  580 \tRWD:  -38.98 \tM_RWD:  120.85 \tLEN:  239\n","ITER:  581 \tRWD:  -48.07 \tM_RWD:  118.02 \tLEN:  156\n","ITER:  582 \tRWD:  10.5 \tM_RWD:  117.82 \tLEN:  389\n","ITER:  583 \tRWD:  238.02 \tM_RWD:  117.24 \tLEN:  384\n","ITER:  584 \tRWD:  -65.81 \tM_RWD:  113.83 \tLEN:  390\n","ITER:  585 \tRWD:  -79.86 \tM_RWD:  110.53 \tLEN:  240\n","ITER:  586 \tRWD:  -58.12 \tM_RWD:  107.39 \tLEN:  258\n","ITER:  587 \tRWD:  -93.84 \tM_RWD:  103.93 \tLEN:  295\n","ITER:  588 \tRWD:  26.98 \tM_RWD:  103.74 \tLEN:  92\n","ITER:  589 \tRWD:  40.8 \tM_RWD:  101.76 \tLEN:  406\n","ITER:  590 \tRWD:  -107.9 \tM_RWD:  98.11 \tLEN:  337\n","ITER:  591 \tRWD:  -105.47 \tM_RWD:  94.76 \tLEN:  352\n","ITER:  592 \tRWD:  -22.88 \tM_RWD:  92.43 \tLEN:  194\n","ITER:  593 \tRWD:  -21.71 \tM_RWD:  90.29 \tLEN:  249\n","ITER:  594 \tRWD:  -40.01 \tM_RWD:  89.45 \tLEN:  234\n","ITER:  595 \tRWD:  221.64 \tM_RWD:  88.87 \tLEN:  271\n","ITER:  596 \tRWD:  -51.26 \tM_RWD:  85.76 \tLEN:  189\n","ITER:  597 \tRWD:  201.51 \tM_RWD:  85.37 \tLEN:  401\n","ITER:  598 \tRWD:  -51.16 \tM_RWD:  82.33 \tLEN:  378\n","ITER:  599 \tRWD:  -112.16 \tM_RWD:  78.63 \tLEN:  289\n","ITER:  600 \tRWD:  173.65 \tM_RWD:  80.03 \tLEN:  321\n","ITER:  601 \tRWD:  254.52 \tM_RWD:  79.86 \tLEN:  488\n","ITER:  602 \tRWD:  232.37 \tM_RWD:  79.68 \tLEN:  860\n","ITER:  603 \tRWD:  209.16 \tM_RWD:  79.07 \tLEN:  553\n","ITER:  604 \tRWD:  -7.46 \tM_RWD:  76.55 \tLEN:  107\n","ITER:  605 \tRWD:  16.52 \tM_RWD:  73.98 \tLEN:  95\n","ITER:  606 \tRWD:  278.44 \tM_RWD:  73.91 \tLEN:  211\n","ITER:  607 \tRWD:  214.43 \tM_RWD:  73.53 \tLEN:  225\n","ITER:  608 \tRWD:  -114.29 \tM_RWD:  70.12 \tLEN:  946\n","ITER:  609 \tRWD:  53.07 \tM_RWD:  67.84 \tLEN:  124\n","ITER:  610 \tRWD:  243.66 \tM_RWD:  67.49 \tLEN:  177\n","ITER:  611 \tRWD:  -187.2 \tM_RWD:  62.82 \tLEN:  683\n","ITER:  612 \tRWD:  180.74 \tM_RWD:  62.21 \tLEN:  388\n","ITER:  613 \tRWD:  -207.53 \tM_RWD:  59.14 \tLEN:  766\n","ITER:  614 \tRWD:  -49.07 \tM_RWD:  55.89 \tLEN:  231\n","ITER:  615 \tRWD:  76.39 \tM_RWD:  53.88 \tLEN:  680\n","ITER:  616 \tRWD:  168.43 \tM_RWD:  52.84 \tLEN:  779\n","ITER:  617 \tRWD:  23.1 \tM_RWD:  50.26 \tLEN:  129\n","ITER:  618 \tRWD:  -66.59 \tM_RWD:  46.67 \tLEN:  238\n","ITER:  619 \tRWD:  -96.87 \tM_RWD:  43.02 \tLEN:  320\n","ITER:  620 \tRWD:  -211.87 \tM_RWD:  38.42 \tLEN:  330\n","ITER:  621 \tRWD:  219.13 \tM_RWD:  37.97 \tLEN:  577\n","ITER:  622 \tRWD:  219.97 \tM_RWD:  42.33 \tLEN:  612\n","ITER:  623 \tRWD:  -96.33 \tM_RWD:  38.69 \tLEN:  1000\n","ITER:  624 \tRWD:  -82.2 \tM_RWD:  37.18 \tLEN:  334\n","ITER:  625 \tRWD:  -500.31 \tM_RWD:  36.25 \tLEN:  665\n","ITER:  626 \tRWD:  -109.27 \tM_RWD:  32.45 \tLEN:  451\n","ITER:  627 \tRWD:  -60.25 \tM_RWD:  37.47 \tLEN:  438\n","ITER:  628 \tRWD:  -50.38 \tM_RWD:  37.17 \tLEN:  199\n","ITER:  629 \tRWD:  -36.19 \tM_RWD:  40.25 \tLEN:  211\n","ITER:  630 \tRWD:  -136.25 \tM_RWD:  38.98 \tLEN:  145\n","ITER:  631 \tRWD:  -160.08 \tM_RWD:  44.16 \tLEN:  1000\n","ITER:  632 \tRWD:  263.78 \tM_RWD:  51.04 \tLEN:  359\n","ITER:  633 \tRWD:  244.84 \tM_RWD:  50.89 \tLEN:  413\n","ITER:  634 \tRWD:  -131.2 \tM_RWD:  55.17 \tLEN:  103\n","ITER:  635 \tRWD:  -302.53 \tM_RWD:  49.68 \tLEN:  178\n","ITER:  636 \tRWD:  200.47 \tM_RWD:  49.13 \tLEN:  469\n","ITER:  637 \tRWD:  -138.24 \tM_RWD:  45.69 \tLEN:  262\n","ITER:  638 \tRWD:  -201.43 \tM_RWD:  46.09 \tLEN:  313\n","ITER:  639 \tRWD:  190.91 \tM_RWD:  45.8 \tLEN:  416\n","ITER:  640 \tRWD:  236.65 \tM_RWD:  45.76 \tLEN:  378\n","ITER:  641 \tRWD:  177.24 \tM_RWD:  45.47 \tLEN:  362\n","ITER:  642 \tRWD:  141.96 \tM_RWD:  48.59 \tLEN:  735\n","ITER:  643 \tRWD:  -87.58 \tM_RWD:  45.5 \tLEN:  107\n","ITER:  644 \tRWD:  -304.95 \tM_RWD:  43.93 \tLEN:  476\n","ITER:  645 \tRWD:  175.83 \tM_RWD:  43.06 \tLEN:  330\n","ITER:  646 \tRWD:  -68.66 \tM_RWD:  39.54 \tLEN:  148\n","ITER:  647 \tRWD:  -124.95 \tM_RWD:  36.45 \tLEN:  175\n","ITER:  648 \tRWD:  -43.96 \tM_RWD:  33.62 \tLEN:  116\n","ITER:  649 \tRWD:  -166.71 \tM_RWD:  29.35 \tLEN:  524\n","ITER:  650 \tRWD:  82.91 \tM_RWD:  27.58 \tLEN:  1000\n","ITER:  651 \tRWD:  -331.3 \tM_RWD:  21.65 \tLEN:  153\n","ITER:  652 \tRWD:  -194.62 \tM_RWD:  21.66 \tLEN:  979\n","ITER:  653 \tRWD:  170.4 \tM_RWD:  25.3 \tLEN:  683\n","ITER:  654 \tRWD:  287.68 \tM_RWD:  30.26 \tLEN:  271\n","ITER:  655 \tRWD:  210.51 \tM_RWD:  29.68 \tLEN:  411\n","ITER:  656 \tRWD:  263.25 \tM_RWD:  32.2 \tLEN:  371\n","ITER:  657 \tRWD:  254.33 \tM_RWD:  32.42 \tLEN:  361\n","ITER:  658 \tRWD:  200.15 \tM_RWD:  31.73 \tLEN:  553\n","ITER:  659 \tRWD:  -54.75 \tM_RWD:  28.71 \tLEN:  1000\n","ITER:  660 \tRWD:  246.31 \tM_RWD:  32.39 \tLEN:  396\n","ITER:  661 \tRWD:  251.89 \tM_RWD:  32.67 \tLEN:  530\n","ITER:  662 \tRWD:  258.29 \tM_RWD:  32.38 \tLEN:  349\n","ITER:  663 \tRWD:  245.64 \tM_RWD:  32.5 \tLEN:  370\n","ITER:  664 \tRWD:  201.22 \tM_RWD:  32.24 \tLEN:  418\n","ITER:  665 \tRWD:  255.45 \tM_RWD:  34.78 \tLEN:  282\n","ITER:  666 \tRWD:  -417.9 \tM_RWD:  34.56 \tLEN:  158\n","ITER:  667 \tRWD:  284.25 \tM_RWD:  34.8 \tLEN:  348\n","ITER:  668 \tRWD:  230.64 \tM_RWD:  35.42 \tLEN:  258\n","ITER:  669 \tRWD:  260.72 \tM_RWD:  36.06 \tLEN:  415\n","ITER:  670 \tRWD:  149.87 \tM_RWD:  35.45 \tLEN:  479\n","ITER:  671 \tRWD:  268.93 \tM_RWD:  38.91 \tLEN:  360\n","ITER:  672 \tRWD:  223.43 \tM_RWD:  39.49 \tLEN:  359\n","ITER:  673 \tRWD:  5.39 \tM_RWD:  40.11 \tLEN:  1000\n","ITER:  674 \tRWD:  190.99 \tM_RWD:  39.48 \tLEN:  855\n","ITER:  675 \tRWD:  249.42 \tM_RWD:  41.98 \tLEN:  342\n","ITER:  676 \tRWD:  233.21 \tM_RWD:  43.9 \tLEN:  292\n","ITER:  677 \tRWD:  232.1 \tM_RWD:  47.09 \tLEN:  437\n","ITER:  678 \tRWD:  262.27 \tM_RWD:  47.33 \tLEN:  338\n","ITER:  679 \tRWD:  223.57 \tM_RWD:  50.39 \tLEN:  230\n","ITER:  680 \tRWD:  -102.84 \tM_RWD:  49.75 \tLEN:  1000\n","ITER:  681 \tRWD:  265.72 \tM_RWD:  52.89 \tLEN:  288\n","ITER:  682 \tRWD:  19.7 \tM_RWD:  52.98 \tLEN:  138\n","ITER:  683 \tRWD:  -69.83 \tM_RWD:  49.91 \tLEN:  296\n","ITER:  684 \tRWD:  8.93 \tM_RWD:  50.65 \tLEN:  106\n","ITER:  685 \tRWD:  298.64 \tM_RWD:  54.44 \tLEN:  413\n","ITER:  686 \tRWD:  250.48 \tM_RWD:  57.52 \tLEN:  426\n","ITER:  687 \tRWD:  199.88 \tM_RWD:  60.46 \tLEN:  548\n","ITER:  688 \tRWD:  185.24 \tM_RWD:  62.04 \tLEN:  180\n","ITER:  689 \tRWD:  175.75 \tM_RWD:  63.39 \tLEN:  389\n","ITER:  690 \tRWD:  159.87 \tM_RWD:  66.07 \tLEN:  342\n","ITER:  691 \tRWD:  -24.31 \tM_RWD:  66.88 \tLEN:  152\n","ITER:  692 \tRWD:  244.23 \tM_RWD:  69.55 \tLEN:  413\n","ITER:  693 \tRWD:  249.27 \tM_RWD:  72.26 \tLEN:  507\n","ITER:  694 \tRWD:  161.4 \tM_RWD:  74.28 \tLEN:  444\n","ITER:  695 \tRWD:  241.75 \tM_RWD:  74.48 \tLEN:  289\n","ITER:  696 \tRWD:  35.43 \tM_RWD:  75.35 \tLEN:  183\n","ITER:  697 \tRWD:  211.57 \tM_RWD:  75.45 \tLEN:  342\n","ITER:  698 \tRWD:  236.94 \tM_RWD:  78.33 \tLEN:  158\n","ITER:  699 \tRWD:  227.38 \tM_RWD:  81.72 \tLEN:  539\n","ITER:  700 \tRWD:  180.31 \tM_RWD:  81.79 \tLEN:  258\n","ITER:  701 \tRWD:  214.86 \tM_RWD:  81.39 \tLEN:  338\n","ITER:  702 \tRWD:  -17.5 \tM_RWD:  78.89 \tLEN:  281\n","ITER:  703 \tRWD:  285.52 \tM_RWD:  79.66 \tLEN:  229\n","ITER:  704 \tRWD:  263.9 \tM_RWD:  82.37 \tLEN:  174\n","ITER:  705 \tRWD:  220.83 \tM_RWD:  84.41 \tLEN:  278\n","ITER:  706 \tRWD:  -448.68 \tM_RWD:  77.14 \tLEN:  482\n","ITER:  707 \tRWD:  219.6 \tM_RWD:  77.2 \tLEN:  169\n","ITER:  708 \tRWD:  272.3 \tM_RWD:  81.06 \tLEN:  300\n","ITER:  709 \tRWD:  3.45 \tM_RWD:  80.57 \tLEN:  103\n","ITER:  710 \tRWD:  208.48 \tM_RWD:  80.21 \tLEN:  171\n","ITER:  711 \tRWD:  291.98 \tM_RWD:  85.01 \tLEN:  200\n","ITER:  712 \tRWD:  255.56 \tM_RWD:  85.75 \tLEN:  164\n","ITER:  713 \tRWD:  12.43 \tM_RWD:  87.95 \tLEN:  94\n","ITER:  714 \tRWD:  236.29 \tM_RWD:  90.81 \tLEN:  177\n","ITER:  715 \tRWD:  272.93 \tM_RWD:  92.77 \tLEN:  269\n","ITER:  716 \tRWD:  230.89 \tM_RWD:  93.4 \tLEN:  573\n","ITER:  717 \tRWD:  261.24 \tM_RWD:  95.78 \tLEN:  189\n","ITER:  718 \tRWD:  257.52 \tM_RWD:  99.02 \tLEN:  195\n","ITER:  719 \tRWD:  254.35 \tM_RWD:  102.53 \tLEN:  244\n","ITER:  720 \tRWD:  217.01 \tM_RWD:  106.82 \tLEN:  247\n","ITER:  721 \tRWD:  223.67 \tM_RWD:  106.87 \tLEN:  254\n","ITER:  722 \tRWD:  218.52 \tM_RWD:  106.85 \tLEN:  366\n","ITER:  723 \tRWD:  -1.18 \tM_RWD:  107.8 \tLEN:  248\n","ITER:  724 \tRWD:  218.48 \tM_RWD:  110.81 \tLEN:  553\n","ITER:  725 \tRWD:  239.79 \tM_RWD:  118.21 \tLEN:  205\n","ITER:  726 \tRWD:  115.29 \tM_RWD:  120.46 \tLEN:  1000\n","ITER:  727 \tRWD:  257.6 \tM_RWD:  123.63 \tLEN:  335\n","ITER:  728 \tRWD:  249.96 \tM_RWD:  126.64 \tLEN:  202\n","ITER:  729 \tRWD:  211.57 \tM_RWD:  129.12 \tLEN:  248\n","ITER:  730 \tRWD:  214.66 \tM_RWD:  132.62 \tLEN:  178\n","ITER:  731 \tRWD:  250.39 \tM_RWD:  136.73 \tLEN:  281\n","ITER:  732 \tRWD:  249.58 \tM_RWD:  136.59 \tLEN:  191\n","ITER:  733 \tRWD:  33.49 \tM_RWD:  134.47 \tLEN:  136\n","ITER:  734 \tRWD:  254.09 \tM_RWD:  138.33 \tLEN:  162\n","ITER:  735 \tRWD:  268.99 \tM_RWD:  144.04 \tLEN:  455\n","ITER:  736 \tRWD:  248.53 \tM_RWD:  144.52 \tLEN:  488\n","ITER:  737 \tRWD:  281.07 \tM_RWD:  148.72 \tLEN:  160\n","ITER:  738 \tRWD:  245.06 \tM_RWD:  153.18 \tLEN:  294\n","ITER:  739 \tRWD:  127.92 \tM_RWD:  152.55 \tLEN:  666\n","ITER:  740 \tRWD:  242.08 \tM_RWD:  152.6 \tLEN:  234\n","ITER:  741 \tRWD:  -171.13 \tM_RWD:  149.12 \tLEN:  262\n","ITER:  742 \tRWD:  272.81 \tM_RWD:  150.43 \tLEN:  208\n","ITER:  743 \tRWD:  276.05 \tM_RWD:  154.07 \tLEN:  345\n","ITER:  744 \tRWD:  259.75 \tM_RWD:  159.71 \tLEN:  194\n","ITER:  745 \tRWD:  -163.75 \tM_RWD:  156.32 \tLEN:  388\n","ITER:  746 \tRWD:  15.38 \tM_RWD:  157.16 \tLEN:  135\n","ITER:  747 \tRWD:  240.25 \tM_RWD:  160.81 \tLEN:  188\n","ITER:  748 \tRWD:  244.13 \tM_RWD:  163.69 \tLEN:  276\n","ITER:  749 \tRWD:  200.1 \tM_RWD:  167.36 \tLEN:  299\n","ITER:  750 \tRWD:  186.57 \tM_RWD:  168.4 \tLEN:  377\n","ITER:  751 \tRWD:  -14.77 \tM_RWD:  171.56 \tLEN:  248\n","ITER:  752 \tRWD:  271.49 \tM_RWD:  176.22 \tLEN:  206\n","ITER:  753 \tRWD:  225.72 \tM_RWD:  176.77 \tLEN:  167\n","ITER:  754 \tRWD:  273.52 \tM_RWD:  176.63 \tLEN:  342\n","ITER:  755 \tRWD:  188.42 \tM_RWD:  176.41 \tLEN:  277\n","ITER:  756 \tRWD:  280.86 \tM_RWD:  176.59 \tLEN:  226\n","ITER:  757 \tRWD:  204.8 \tM_RWD:  176.09 \tLEN:  493\n","ITER:  758 \tRWD:  258.58 \tM_RWD:  176.68 \tLEN:  154\n","ITER:  759 \tRWD:  247.94 \tM_RWD:  179.7 \tLEN:  195\n","ITER:  760 \tRWD:  238.74 \tM_RWD:  179.63 \tLEN:  793\n","ITER:  761 \tRWD:  270.02 \tM_RWD:  179.81 \tLEN:  339\n","ITER:  762 \tRWD:  260.64 \tM_RWD:  179.83 \tLEN:  298\n","ITER:  763 \tRWD:  154.71 \tM_RWD:  178.92 \tLEN:  624\n","ITER:  764 \tRWD:  185.09 \tM_RWD:  178.76 \tLEN:  583\n","ITER:  765 \tRWD:  -36.5 \tM_RWD:  175.84 \tLEN:  157\n","ITER:  766 \tRWD:  245.01 \tM_RWD:  182.47 \tLEN:  307\n","ITER:  767 \tRWD:  270.44 \tM_RWD:  182.33 \tLEN:  345\n","ITER:  768 \tRWD:  -131.52 \tM_RWD:  178.71 \tLEN:  1000\n","ITER:  769 \tRWD:  206.53 \tM_RWD:  178.17 \tLEN:  441\n","ITER:  770 \tRWD:  119.87 \tM_RWD:  177.87 \tLEN:  301\n","ITER:  771 \tRWD:  263.41 \tM_RWD:  177.82 \tLEN:  308\n","ITER:  772 \tRWD:  220.85 \tM_RWD:  177.79 \tLEN:  361\n","ITER:  773 \tRWD:  194.94 \tM_RWD:  179.69 \tLEN:  230\n","ITER:  774 \tRWD:  152.57 \tM_RWD:  179.3 \tLEN:  460\n","ITER:  775 \tRWD:  195.08 \tM_RWD:  178.76 \tLEN:  332\n","ITER:  776 \tRWD:  239.87 \tM_RWD:  178.82 \tLEN:  203\n","ITER:  777 \tRWD:  0.08 \tM_RWD:  176.5 \tLEN:  126\n","ITER:  778 \tRWD:  -146.95 \tM_RWD:  172.41 \tLEN:  407\n","ITER:  779 \tRWD:  -89.26 \tM_RWD:  169.28 \tLEN:  364\n","ITER:  780 \tRWD:  -231.26 \tM_RWD:  168.0 \tLEN:  345\n","ITER:  781 \tRWD:  -19.9 \tM_RWD:  165.14 \tLEN:  207\n","ITER:  782 \tRWD:  -28.0 \tM_RWD:  164.67 \tLEN:  150\n","ITER:  783 \tRWD:  143.11 \tM_RWD:  166.8 \tLEN:  500\n","ITER:  784 \tRWD:  -68.42 \tM_RWD:  166.02 \tLEN:  158\n","ITER:  785 \tRWD:  233.66 \tM_RWD:  165.37 \tLEN:  192\n","ITER:  786 \tRWD:  181.64 \tM_RWD:  164.68 \tLEN:  337\n","ITER:  787 \tRWD:  -38.34 \tM_RWD:  162.3 \tLEN:  370\n","ITER:  788 \tRWD:  -66.54 \tM_RWD:  159.78 \tLEN:  217\n","ITER:  789 \tRWD:  218.7 \tM_RWD:  160.21 \tLEN:  247\n","ITER:  790 \tRWD:  193.92 \tM_RWD:  160.55 \tLEN:  647\n","ITER:  791 \tRWD:  -51.96 \tM_RWD:  160.28 \tLEN:  265\n","ITER:  792 \tRWD:  256.54 \tM_RWD:  160.4 \tLEN:  282\n","ITER:  793 \tRWD:  163.26 \tM_RWD:  159.54 \tLEN:  454\n","ITER:  794 \tRWD:  251.55 \tM_RWD:  160.44 \tLEN:  855\n","ITER:  795 \tRWD:  -187.99 \tM_RWD:  156.15 \tLEN:  261\n","ITER:  796 \tRWD:  214.81 \tM_RWD:  157.94 \tLEN:  203\n","ITER:  797 \tRWD:  -59.69 \tM_RWD:  155.23 \tLEN:  351\n","ITER:  798 \tRWD:  -79.64 \tM_RWD:  152.06 \tLEN:  251\n","ITER:  799 \tRWD:  -168.6 \tM_RWD:  148.1 \tLEN:  596\n","ITER:  800 \tRWD:  -12.52 \tM_RWD:  146.17 \tLEN:  223\n","ITER:  801 \tRWD:  -97.12 \tM_RWD:  143.05 \tLEN:  234\n","ITER:  802 \tRWD:  223.97 \tM_RWD:  145.47 \tLEN:  397\n","ITER:  803 \tRWD:  157.12 \tM_RWD:  144.18 \tLEN:  417\n","ITER:  804 \tRWD:  219.68 \tM_RWD:  143.74 \tLEN:  267\n","ITER:  805 \tRWD:  -10.69 \tM_RWD:  141.43 \tLEN:  254\n","ITER:  806 \tRWD:  181.41 \tM_RWD:  147.73 \tLEN:  510\n","ITER:  807 \tRWD:  153.29 \tM_RWD:  147.06 \tLEN:  647\n","ITER:  808 \tRWD:  255.02 \tM_RWD:  146.89 \tLEN:  270\n","ITER:  809 \tRWD:  131.56 \tM_RWD:  148.17 \tLEN:  777\n","ITER:  810 \tRWD:  243.58 \tM_RWD:  148.52 \tLEN:  171\n","ITER:  811 \tRWD:  -339.97 \tM_RWD:  142.2 \tLEN:  828\n","ITER:  812 \tRWD:  269.43 \tM_RWD:  142.34 \tLEN:  176\n","ITER:  813 \tRWD:  241.95 \tM_RWD:  144.64 \tLEN:  175\n","ITER:  814 \tRWD:  224.49 \tM_RWD:  144.52 \tLEN:  228\n","ITER:  815 \tRWD:  144.01 \tM_RWD:  143.23 \tLEN:  522\n","ITER:  816 \tRWD:  233.0 \tM_RWD:  143.25 \tLEN:  162\n","ITER:  817 \tRWD:  204.7 \tM_RWD:  142.69 \tLEN:  173\n","ITER:  818 \tRWD:  200.88 \tM_RWD:  142.12 \tLEN:  171\n","ITER:  819 \tRWD:  226.3 \tM_RWD:  141.84 \tLEN:  206\n","ITER:  820 \tRWD:  229.92 \tM_RWD:  141.97 \tLEN:  332\n","ITER:  821 \tRWD:  188.31 \tM_RWD:  141.61 \tLEN:  177\n","ITER:  822 \tRWD:  257.04 \tM_RWD:  142.0 \tLEN:  197\n","ITER:  823 \tRWD:  -85.07 \tM_RWD:  141.16 \tLEN:  191\n","ITER:  824 \tRWD:  225.5 \tM_RWD:  141.23 \tLEN:  228\n","ITER:  825 \tRWD:  162.07 \tM_RWD:  140.45 \tLEN:  322\n","ITER:  826 \tRWD:  198.16 \tM_RWD:  141.28 \tLEN:  361\n","ITER:  827 \tRWD:  226.31 \tM_RWD:  140.97 \tLEN:  215\n","ITER:  828 \tRWD:  -85.58 \tM_RWD:  137.61 \tLEN:  212\n","ITER:  829 \tRWD:  -54.46 \tM_RWD:  134.95 \tLEN:  204\n","ITER:  830 \tRWD:  -101.83 \tM_RWD:  131.79 \tLEN:  334\n","ITER:  831 \tRWD:  133.87 \tM_RWD:  130.62 \tLEN:  772\n","ITER:  832 \tRWD:  223.48 \tM_RWD:  130.36 \tLEN:  420\n","ITER:  833 \tRWD:  235.43 \tM_RWD:  132.38 \tLEN:  182\n","ITER:  834 \tRWD:  182.37 \tM_RWD:  131.66 \tLEN:  737\n","ITER:  835 \tRWD:  158.88 \tM_RWD:  130.56 \tLEN:  583\n","ITER:  836 \tRWD:  218.66 \tM_RWD:  130.27 \tLEN:  670\n","ITER:  837 \tRWD:  258.26 \tM_RWD:  130.04 \tLEN:  608\n","ITER:  838 \tRWD:  267.7 \tM_RWD:  130.26 \tLEN:  221\n","ITER:  839 \tRWD:  212.53 \tM_RWD:  131.11 \tLEN:  729\n","ITER:  840 \tRWD:  217.6 \tM_RWD:  130.86 \tLEN:  749\n","ITER:  841 \tRWD:  229.52 \tM_RWD:  134.87 \tLEN:  492\n","ITER:  842 \tRWD:  -80.77 \tM_RWD:  131.34 \tLEN:  1000\n","ITER:  843 \tRWD:  -19.75 \tM_RWD:  128.38 \tLEN:  253\n","ITER:  844 \tRWD:  235.5 \tM_RWD:  128.13 \tLEN:  837\n","ITER:  845 \tRWD:  287.85 \tM_RWD:  132.65 \tLEN:  234\n","ITER:  846 \tRWD:  251.29 \tM_RWD:  135.01 \tLEN:  609\n","ITER:  847 \tRWD:  141.36 \tM_RWD:  134.02 \tLEN:  749\n","ITER:  848 \tRWD:  -139.3 \tM_RWD:  130.19 \tLEN:  441\n","ITER:  849 \tRWD:  -2.67 \tM_RWD:  128.16 \tLEN:  99\n","ITER:  850 \tRWD:  182.8 \tM_RWD:  128.12 \tLEN:  561\n","ITER:  851 \tRWD:  228.8 \tM_RWD:  130.56 \tLEN:  615\n","ITER:  852 \tRWD:  12.25 \tM_RWD:  127.96 \tLEN:  248\n","ITER:  853 \tRWD:  197.17 \tM_RWD:  127.68 \tLEN:  288\n","ITER:  854 \tRWD:  246.27 \tM_RWD:  127.41 \tLEN:  257\n","ITER:  855 \tRWD:  300.52 \tM_RWD:  128.53 \tLEN:  168\n","ITER:  856 \tRWD:  310.95 \tM_RWD:  128.83 \tLEN:  206\n","ITER:  857 \tRWD:  227.22 \tM_RWD:  129.05 \tLEN:  336\n","ITER:  858 \tRWD:  190.4 \tM_RWD:  128.37 \tLEN:  273\n","ITER:  859 \tRWD:  241.5 \tM_RWD:  128.31 \tLEN:  446\n","ITER:  860 \tRWD:  187.21 \tM_RWD:  127.79 \tLEN:  521\n","ITER:  861 \tRWD:  159.55 \tM_RWD:  126.69 \tLEN:  259\n","ITER:  862 \tRWD:  243.63 \tM_RWD:  126.52 \tLEN:  339\n","ITER:  863 \tRWD:  170.02 \tM_RWD:  126.67 \tLEN:  427\n","ITER:  864 \tRWD:  165.02 \tM_RWD:  126.47 \tLEN:  431\n","ITER:  865 \tRWD:  -29.3 \tM_RWD:  126.54 \tLEN:  284\n","ITER:  866 \tRWD:  251.44 \tM_RWD:  126.61 \tLEN:  298\n","ITER:  867 \tRWD:  205.24 \tM_RWD:  125.95 \tLEN:  513\n","ITER:  868 \tRWD:  271.83 \tM_RWD:  129.99 \tLEN:  199\n","ITER:  869 \tRWD:  6.07 \tM_RWD:  127.98 \tLEN:  129\n","ITER:  870 \tRWD:  227.91 \tM_RWD:  129.06 \tLEN:  546\n","ITER:  871 \tRWD:  229.63 \tM_RWD:  128.72 \tLEN:  234\n","ITER:  872 \tRWD:  21.98 \tM_RWD:  126.74 \tLEN:  145\n","ITER:  873 \tRWD:  213.26 \tM_RWD:  126.92 \tLEN:  253\n","ITER:  874 \tRWD:  240.1 \tM_RWD:  127.79 \tLEN:  369\n","ITER:  875 \tRWD:  216.94 \tM_RWD:  128.01 \tLEN:  388\n","ITER:  876 \tRWD:  270.1 \tM_RWD:  128.32 \tLEN:  266\n","ITER:  877 \tRWD:  257.52 \tM_RWD:  130.89 \tLEN:  838\n","ITER:  878 \tRWD:  251.39 \tM_RWD:  134.87 \tLEN:  224\n","ITER:  879 \tRWD:  295.54 \tM_RWD:  138.72 \tLEN:  205\n","ITER:  880 \tRWD:  232.99 \tM_RWD:  143.36 \tLEN:  194\n","ITER:  881 \tRWD:  276.36 \tM_RWD:  146.33 \tLEN:  204\n","ITER:  882 \tRWD:  248.91 \tM_RWD:  149.1 \tLEN:  472\n","ITER:  883 \tRWD:  226.83 \tM_RWD:  149.93 \tLEN:  219\n","ITER:  884 \tRWD:  264.64 \tM_RWD:  153.26 \tLEN:  218\n","ITER:  885 \tRWD:  235.3 \tM_RWD:  153.28 \tLEN:  214\n","ITER:  886 \tRWD:  267.2 \tM_RWD:  154.14 \tLEN:  349\n","ITER:  887 \tRWD:  243.49 \tM_RWD:  156.95 \tLEN:  192\n","ITER:  888 \tRWD:  -369.62 \tM_RWD:  153.92 \tLEN:  186\n","ITER:  889 \tRWD:  242.28 \tM_RWD:  154.16 \tLEN:  198\n","ITER:  890 \tRWD:  219.4 \tM_RWD:  154.41 \tLEN:  226\n","ITER:  891 \tRWD:  254.99 \tM_RWD:  157.48 \tLEN:  180\n","ITER:  892 \tRWD:  257.9 \tM_RWD:  157.5 \tLEN:  560\n","ITER:  893 \tRWD:  274.82 \tM_RWD:  158.61 \tLEN:  441\n","ITER:  894 \tRWD:  270.93 \tM_RWD:  158.81 \tLEN:  184\n","ITER:  895 \tRWD:  244.6 \tM_RWD:  163.13 \tLEN:  212\n","ITER:  896 \tRWD:  305.47 \tM_RWD:  164.04 \tLEN:  157\n","ITER:  897 \tRWD:  268.06 \tM_RWD:  167.32 \tLEN:  222\n","ITER:  898 \tRWD:  214.04 \tM_RWD:  170.25 \tLEN:  261\n","ITER:  899 \tRWD:  265.88 \tM_RWD:  174.6 \tLEN:  206\n","ITER:  900 \tRWD:  229.93 \tM_RWD:  177.02 \tLEN:  400\n","ITER:  901 \tRWD:  285.36 \tM_RWD:  180.85 \tLEN:  248\n","ITER:  902 \tRWD:  238.56 \tM_RWD:  180.99 \tLEN:  224\n","ITER:  903 \tRWD:  245.27 \tM_RWD:  181.87 \tLEN:  227\n","ITER:  904 \tRWD:  226.91 \tM_RWD:  181.95 \tLEN:  264\n","ITER:  905 \tRWD:  238.08 \tM_RWD:  184.43 \tLEN:  236\n","ITER:  906 \tRWD:  252.34 \tM_RWD:  185.14 \tLEN:  196\n","ITER:  907 \tRWD:  261.52 \tM_RWD:  186.23 \tLEN:  202\n","ITER:  908 \tRWD:  62.5 \tM_RWD:  184.3 \tLEN:  1000\n","ITER:  909 \tRWD:  273.27 \tM_RWD:  185.72 \tLEN:  224\n","ITER:  910 \tRWD:  264.55 \tM_RWD:  185.93 \tLEN:  238\n","ITER:  911 \tRWD:  190.01 \tM_RWD:  191.23 \tLEN:  289\n","ITER:  912 \tRWD:  261.99 \tM_RWD:  191.15 \tLEN:  278\n","ITER:  913 \tRWD:  272.37 \tM_RWD:  191.46 \tLEN:  294\n","ITER:  914 \tRWD:  291.19 \tM_RWD:  192.12 \tLEN:  221\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  915 \tRWD:  293.49 \tM_RWD:  193.62 \tLEN:  210\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  916 \tRWD:  239.47 \tM_RWD:  193.68 \tLEN:  376\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  917 \tRWD:  268.92 \tM_RWD:  194.33 \tLEN:  234\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  918 \tRWD:  239.37 \tM_RWD:  194.71 \tLEN:  288\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  919 \tRWD:  263.59 \tM_RWD:  195.08 \tLEN:  171\n","ITER:  920 \tRWD:  -487.47 \tM_RWD:  187.91 \tLEN:  98\n","ITER:  921 \tRWD:  277.19 \tM_RWD:  188.8 \tLEN:  288\n","ITER:  922 \tRWD:  277.18 \tM_RWD:  189.0 \tLEN:  292\n","ITER:  923 \tRWD:  260.43 \tM_RWD:  192.45 \tLEN:  175\n","ITER:  924 \tRWD:  277.85 \tM_RWD:  192.98 \tLEN:  189\n","ITER:  925 \tRWD:  268.87 \tM_RWD:  194.05 \tLEN:  331\n","ITER:  926 \tRWD:  241.43 \tM_RWD:  194.48 \tLEN:  305\n","ITER:  927 \tRWD:  254.05 \tM_RWD:  194.76 \tLEN:  410\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  928 \tRWD:  251.38 \tM_RWD:  198.13 \tLEN:  145\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  929 \tRWD:  266.74 \tM_RWD:  201.34 \tLEN:  425\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  930 \tRWD:  247.44 \tM_RWD:  204.83 \tLEN:  246\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  931 \tRWD:  265.07 \tM_RWD:  206.14 \tLEN:  212\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  932 \tRWD:  235.11 \tM_RWD:  206.26 \tLEN:  185\n","ITER:  933 \tRWD:  -257.01 \tM_RWD:  201.33 \tLEN:  131\n","ITER:  934 \tRWD:  -271.11 \tM_RWD:  196.8 \tLEN:  114\n","ITER:  935 \tRWD:  -283.12 \tM_RWD:  192.38 \tLEN:  140\n","ITER:  936 \tRWD:  268.38 \tM_RWD:  192.88 \tLEN:  141\n","ITER:  937 \tRWD:  249.86 \tM_RWD:  192.79 \tLEN:  179\n","ITER:  938 \tRWD:  257.85 \tM_RWD:  192.69 \tLEN:  323\n","ITER:  939 \tRWD:  258.12 \tM_RWD:  193.15 \tLEN:  177\n","ITER:  940 \tRWD:  272.75 \tM_RWD:  193.7 \tLEN:  156\n","ITER:  941 \tRWD:  276.09 \tM_RWD:  194.17 \tLEN:  318\n","ITER:  942 \tRWD:  252.37 \tM_RWD:  197.5 \tLEN:  363\n","ITER:  943 \tRWD:  251.5 \tM_RWD:  200.21 \tLEN:  299\n","ITER:  944 \tRWD:  273.73 \tM_RWD:  200.59 \tLEN:  192\n","ITER:  945 \tRWD:  262.94 \tM_RWD:  200.34 \tLEN:  443\n","ITER:  946 \tRWD:  271.23 \tM_RWD:  200.54 \tLEN:  258\n","ITER:  947 \tRWD:  276.9 \tM_RWD:  201.9 \tLEN:  307\n","ITER:  948 \tRWD:  264.44 \tM_RWD:  205.94 \tLEN:  216\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  949 \tRWD:  260.0 \tM_RWD:  208.56 \tLEN:  168\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  950 \tRWD:  267.01 \tM_RWD:  209.41 \tLEN:  271\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  951 \tRWD:  271.98 \tM_RWD:  209.84 \tLEN:  305\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  952 \tRWD:  234.74 \tM_RWD:  212.06 \tLEN:  215\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  953 \tRWD:  288.74 \tM_RWD:  212.98 \tLEN:  253\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  954 \tRWD:  264.13 \tM_RWD:  213.16 \tLEN:  225\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  955 \tRWD:  301.41 \tM_RWD:  213.17 \tLEN:  216\n","ITER:  956 \tRWD:  284.63 \tM_RWD:  212.9 \tLEN:  235\n","ITER:  957 \tRWD:  159.75 \tM_RWD:  212.23 \tLEN:  447\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  958 \tRWD:  286.22 \tM_RWD:  213.19 \tLEN:  189\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  959 \tRWD:  280.77 \tM_RWD:  213.58 \tLEN:  170\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  960 \tRWD:  281.86 \tM_RWD:  214.52 \tLEN:  187\n","ITER:  961 \tRWD:  -67.6 \tM_RWD:  212.25 \tLEN:  131\n","ITER:  962 \tRWD:  256.82 \tM_RWD:  212.39 \tLEN:  199\n","ITER:  963 \tRWD:  230.03 \tM_RWD:  212.99 \tLEN:  207\n","ITER:  964 \tRWD:  292.01 \tM_RWD:  214.26 \tLEN:  242\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  965 \tRWD:  285.64 \tM_RWD:  217.4 \tLEN:  188\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  966 \tRWD:  283.7 \tM_RWD:  217.73 \tLEN:  168\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  967 \tRWD:  241.78 \tM_RWD:  218.09 \tLEN:  174\n","ITER:  968 \tRWD:  263.65 \tM_RWD:  218.01 \tLEN:  233\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  969 \tRWD:  310.25 \tM_RWD:  221.05 \tLEN:  288\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  970 \tRWD:  253.21 \tM_RWD:  221.31 \tLEN:  211\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  971 \tRWD:  262.71 \tM_RWD:  221.64 \tLEN:  243\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  972 \tRWD:  270.12 \tM_RWD:  224.12 \tLEN:  216\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  973 \tRWD:  264.2 \tM_RWD:  224.63 \tLEN:  153\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  974 \tRWD:  294.72 \tM_RWD:  225.17 \tLEN:  383\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  975 \tRWD:  277.36 \tM_RWD:  225.78 \tLEN:  187\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  976 \tRWD:  313.2 \tM_RWD:  226.21 \tLEN:  270\n","ITER:  977 \tRWD:  253.2 \tM_RWD:  226.17 \tLEN:  366\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  978 \tRWD:  303.47 \tM_RWD:  226.69 \tLEN:  258\n","ITER:  979 \tRWD:  243.47 \tM_RWD:  226.17 \tLEN:  158\n","ITER:  980 \tRWD:  282.9 \tM_RWD:  226.66 \tLEN:  344\n","ITER:  981 \tRWD:  276.43 \tM_RWD:  226.67 \tLEN:  158\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  982 \tRWD:  277.42 \tM_RWD:  226.95 \tLEN:  166\n","ITER:  983 \tRWD:  222.05 \tM_RWD:  226.9 \tLEN:  189\n","ITER:  984 \tRWD:  62.79 \tM_RWD:  224.88 \tLEN:  107\n","ITER:  985 \tRWD:  308.92 \tM_RWD:  225.62 \tLEN:  234\n","ITER:  986 \tRWD:  252.14 \tM_RWD:  225.47 \tLEN:  185\n","ITER:  987 \tRWD:  298.61 \tM_RWD:  226.02 \tLEN:  298\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  988 \tRWD:  273.96 \tM_RWD:  232.46 \tLEN:  180\n","[INFO] ACTOR - Saving model\n","[INFO] ACTOR - Saving model\n","[INFO] CRITIC - Saving model\n","[INFO] CRITIC - Saving model\n","ITER:  989 \tRWD:  263.17 \tM_RWD:  232.67 \tLEN:  157\n","ITER:  990 \tRWD:  34.31 \tM_RWD:  230.81 \tLEN:  131\n","ITER:  991 \tRWD:  235.22 \tM_RWD:  230.62 \tLEN:  191\n","ITER:  992 \tRWD:  245.45 \tM_RWD:  230.49 \tLEN:  213\n","ITER:  993 \tRWD:  267.77 \tM_RWD:  230.42 \tLEN:  142\n","ITER:  994 \tRWD:  247.85 \tM_RWD:  230.19 \tLEN:  207\n","ITER:  995 \tRWD:  61.66 \tM_RWD:  228.36 \tLEN:  190\n","ITER:  996 \tRWD:  299.1 \tM_RWD:  228.3 \tLEN:  195\n","ITER:  997 \tRWD:  240.66 \tM_RWD:  228.02 \tLEN:  706\n","ITER:  998 \tRWD:  44.59 \tM_RWD:  226.33 \tLEN:  115\n","ITER:  999 \tRWD:  265.86 \tM_RWD:  226.33 \tLEN:  186\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"gADvBLe7VySC","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1598142087824,"user_tz":-330,"elapsed":5845772,"user":{"displayName":"Nimish S","photoUrl":"","userId":"16860995465865824316"}}},"source":["plt.plot(mean_rewards)"],"execution_count":11,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"9VHOmd5dWHgZ","colab_type":"text"},"source":["## **Testing**"]},{"cell_type":"code","metadata":{"id":"R1wIIZygWOZj","colab_type":"code","colab":{}},"source":["# env_name = \"LunarLanderContinuous-v2\"\n","# env = gym.make(env_name)\n","\n","# agent = DDPGAgent(observation_shape=env.observation_space.shape,\n","#                   n_actions=env.action_space.shape[0],\n","#                   actor_lr=1e-4,\n","#                   critic_lr=1e-3,\n","#                   fc1_size=400,\n","#                   fc2_size=300,\n","#                   tau=0.001,\n","#                   gamma=0.99,\n","#                   mem_size=1,\n","#                   batch_size=1,\n","#                   algo_name='DDPG',\n","#                   env_name=env_name,\n","#                   model_dir='./')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OX5IXyagWGms","colab_type":"code","colab":{}},"source":["#model.eval()\n","# with T.no_grad():\n","#     total_reward, total_moves = 0,0\n","#     done = False\n","#     observation = env.reset()\n","\n","#     while not done:\n","#         time.sleep(0.0001)\n","#         env.render()\n","\n","#         # e_GREEDY ACTION\n","#         action = agent.get_action(observation)\n","#         observation_, reward, done, _ = env.step(action)\n","\n","#         total_reward += reward\n","#         total_moves += 1\n","\n","#         observation = observation_\n","#     print(\"RWD: \",total_reward,\"\\tLEN: \",total_moves)\n","#     env.close()"],"execution_count":null,"outputs":[]}]}