training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(True, '2024년 02월 22일 15:54:20')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import yaml\n",
    "import datetime\n",
    "import torch\n",
    "from myutils import get_directory_path, configmake, make_weight_dict, make_type_dict, make_env, change_model\n",
    "dir_path  = get_directory_path()\n",
    "torch.cuda.is_available() , datetime.datetime.now().strftime(\"%Y년 %m월 %d일 %H:%M:%S\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train 1622"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(r'config\\\\scone_walk_h1622.yaml','r') as y:\n",
    "    config = yaml.safe_load(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "설정을 바꾸지 않고 종료합니다.\n"
     ]
    }
   ],
   "source": [
    "config = configmake(config,skip_while_loop=True)\n",
    "config = make_weight_dict(config,is_weight=False)\n",
    "config = make_type_dict(config,is_weight=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'deprl.custom_trainer.Trainer(steps=int(5e8), epoch_steps=int(2e5), save_steps=int(1e6))'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config['tonic']['trainer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#config['tonic']['name'] = 'balance_0504'\n",
    "# config['tonic']['name']  = \"noLateral\"\n",
    "# config['tonic']['name']  = \"noLateral_v2\"\n",
    "# config['tonic']['name']  = \"combine_change_vel\"\n",
    "# config['tonic']['name']  = \"combine_range\"\n",
    "# config['tonic']['name']  = \"Hard_balance\"\n",
    "# config['tonic']['name']  = \"default_time\"\n",
    "# config['tonic']['name']  = \"default_time_v2\"\n",
    "config['tonic']['name']  = \"temp\"\n",
    "epoch = 500\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# config['weights']['type']['grfreduce'] = 0.005\n",
    "# config['weights']['reward']['grf_grfdelta'] = -0.7\n",
    "# config['weights']['reward']['grf_grfdelta_x'] = -0.3\n",
    "\n",
    "# config['weights']['type']['balance'] = 0.5\n",
    "# config['weights']['reward']['balance_diff_position_z'] = -0.6\n",
    "# config['weights']['reward']['balance_position_z'] = -0.4\n",
    "\n",
    "config['weights']['type']['alive'] = 5\n",
    "config['weights']['reward']['alive_livetime'] = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"deprl.environments.Gym('sconewalk_h1622-v1')\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# config = change_model(config,'sconewalk_noLateral-v2')\n",
    "config['tonic']['environment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config 파일 저장 완료\n"
     ]
    }
   ],
   "source": [
    "config = make_env(config)\n",
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"deprl.environments.Gym('sconewalk_h1622-v1', rwd_type_weights={'alive': 5},rwd_weights={'alive_livetime': 1})\""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config['tonic']['environment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weight가 변경되었습니다.\n",
      "type_weight를 보여줍니다.\n",
      "             Key  Value\n",
      "0          alive    5.0\n",
      "1        balance    0.0\n",
      "2     deprlpaper    0.0\n",
      "3      grfreduce    0.0\n",
      "4     measureLua    0.0\n",
      "5  trunk_balance    0.0\n",
      "변경된 reward 가중치를 보여줍니다.\n",
      "              Key  Value\n",
      "0  alive_livetime      1\n"
     ]
    }
   ],
   "source": [
    "from main import main\n",
    "main(config,setting=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DEFALT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(r'config\\\\scone_walk_h1622.yaml','r') as y:\n",
    "    config = yaml.safe_load(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = configmake(config,skip_while_loop=True)\n",
    "config = make_weight_dict(config,is_weight=False)\n",
    "config = make_type_dict(config,is_weight=False)\n",
    "config['tonic']['trainer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['name']  = \"temp\"\n",
    "epoch = 500\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = make_env(config)\n",
    "print(config['tonic']['environment'])\n",
    "config = change_model(config,'sconewalk_h1622-range-v1',change = True)\n",
    "print(config['tonic']['environment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from main import main\n",
    "main(config,setting=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train 1922"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(r'config\\\\scone_walk_h1922.yaml','r') as y:\n",
    "    config = yaml.safe_load(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = configmake(config,skip_while_loop=True)\n",
    "config = make_weight_dict(config,is_weight=False)\n",
    "config = make_type_dict(config,is_weight=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['trainer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#config['tonic']['name'] = 'balance_0504'\n",
    "# config['tonic']['name']  = \"noLateral\"\n",
    "# config['tonic']['name']  = \"noLateral_v2\"\n",
    "# config['tonic']['name']  = \"change_range\"\n",
    "# config['tonic']['name']  = \"change_gym_100n\"\n",
    "# config['tonic']['name']  = \"activation_layer\"\n",
    "config['tonic']['name']  = \"change_gym_30n\"\n",
    "# config['tonic']['name']  = \"temp10\"\n",
    "epoch = 500\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# config['weights']['type']['grfreduce'] = 0.005\n",
    "config['weights']['type']['balance'] = 2\n",
    "# #config['weights']['reward']['grf_grfdelta'] = -0.7\n",
    "# # config['weights']['reward']['grf_grfdelta_x'] = -0.3\n",
    "config['weights']['reward']['balance_velocity_zcom'] = -0.3\n",
    "config['weights']['reward']['balance_velcotiy_zhead'] = -0.7\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# config = change_model(config,'sconewalk_noLateral-v2')\n",
    "config['tonic']['environment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = make_env(config)\n",
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['environment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from main import main\n",
    "main(config,setting=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DEFALT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = configmake(config,skip_while_loop=True)\n",
    "config = make_weight_dict(config,is_weight=False)\n",
    "config = make_type_dict(config,is_weight=False)\n",
    "config['tonic']['trainer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['name']  = \"default_vel_new\"\n",
    "epoch = 500\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = make_env(config)\n",
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from main import main\n",
    "main(config,setting=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RETRAIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"combine_change_vel\"\n",
    "with open(f'./trainconfig/{name}.yaml','r') as y:\n",
    "    config = yaml.safe_load(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 500\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'\n",
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from main import main\n",
    "main(config,setting=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train 1622_harness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(r'config\\\\scone_walk_harness_h1622.yaml','r') as y:\n",
    "    config = yaml.safe_load(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "설정을 바꾸지 않고 종료합니다.\n"
     ]
    }
   ],
   "source": [
    "config = configmake(config,skip_while_loop=True)\n",
    "config = make_weight_dict(config,is_weight=False)\n",
    "config = make_type_dict(config,is_weight=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['name']  = \"harness\"\n",
    "epoch = 500\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "환경을 바꾸지 않고 종료합니다.\n",
      "config 파일 저장 완료\n"
     ]
    }
   ],
   "source": [
    "config = make_env(config)\n",
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"deprl.environments.Gym('sconewalk_harness_h1622-v1')\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config['tonic']['environment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weight가 변경되었습니다.\n",
      "type_weight를 보여줍니다.\n",
      "customreward 함수를 사용하지 않았습니다.\n",
      "reward weight를 변경하지 않았습니다.\n"
     ]
    },
    {
     "ename": "UnregisteredEnv",
     "evalue": "No registered env with id: sconewalk_harness_h1622-v1",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\gym\\envs\\registration.py:132\u001b[0m, in \u001b[0;36mEnvRegistry.spec\u001b[1;34m(self, path)\u001b[0m\n\u001b[0;32m    131\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 132\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv_specs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mid\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[0;32m    133\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[0;32m    134\u001b[0m     \u001b[38;5;66;03m# Parse the env name and check to see if it matches the non-version\u001b[39;00m\n\u001b[0;32m    135\u001b[0m     \u001b[38;5;66;03m# part of a valid env (could also check the exact number here)\u001b[39;00m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'sconewalk_harness_h1622-v1'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mUnregisteredEnv\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[7], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmain\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m main\n\u001b[1;32m----> 2\u001b[0m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43msetting\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\main.py:17\u001b[0m, in \u001b[0;36mmain\u001b[1;34m(config, setting)\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m     16\u001b[0m     set_tensor_device()\n\u001b[1;32m---> 17\u001b[0m     \u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43msetting\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43msetting\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\main.py:40\u001b[0m, in \u001b[0;36mtrain\u001b[1;34m(config, setting)\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[38;5;66;03m# Build the training environment.\u001b[39;00m\n\u001b[0;32m     39\u001b[0m _environment \u001b[38;5;241m=\u001b[39m tonic_conf[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menvironment\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m---> 40\u001b[0m environment \u001b[38;5;241m=\u001b[39m \u001b[43mcustom_distributed\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdistribute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     41\u001b[0m \u001b[43m    \u001b[49m\u001b[43menvironment\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_environment\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     42\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtonic_conf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtonic_conf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     43\u001b[0m \u001b[43m    \u001b[49m\u001b[43menv_args\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43menv_args\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     44\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     45\u001b[0m environment\u001b[38;5;241m.\u001b[39minitialize(seed\u001b[38;5;241m=\u001b[39mtonic_conf[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mseed\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[0;32m     47\u001b[0m \u001b[38;5;66;03m# Build the testing environment.\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\custom_distributed.py:278\u001b[0m, in \u001b[0;36mdistribute\u001b[1;34m(environment, tonic_conf, env_args, parallel, sequential)\u001b[0m\n\u001b[0;32m    273\u001b[0m sequential \u001b[38;5;241m=\u001b[39m tonic_conf[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msequential\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m sequential \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m sequential\n\u001b[0;32m    274\u001b[0m build_dict \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(\n\u001b[0;32m    275\u001b[0m     env\u001b[38;5;241m=\u001b[39menvironment, parallel\u001b[38;5;241m=\u001b[39mparallel, sequential\u001b[38;5;241m=\u001b[39msequential\n\u001b[0;32m    276\u001b[0m )\n\u001b[1;32m--> 278\u001b[0m dummy_environment \u001b[38;5;241m=\u001b[39m \u001b[43mbuild_env_from_dict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuild_dict\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    279\u001b[0m max_episode_steps \u001b[38;5;241m=\u001b[39m dummy_environment\u001b[38;5;241m.\u001b[39m_max_episode_steps\n\u001b[0;32m    280\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m dummy_environment\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\custom_distributed.py:305\u001b[0m, in \u001b[0;36mbuild_env_from_dict\u001b[1;34m(build_dict)\u001b[0m\n\u001b[0;32m    302\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(build_dict) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mdict\u001b[39m:\n\u001b[0;32m    303\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdeprl\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m env_tonic_compat\n\u001b[1;32m--> 305\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m env_tonic_compat(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mbuild_dict)\n\u001b[0;32m    306\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    307\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m build_dict()\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\env_wrappers\\__init__.py:26\u001b[0m, in \u001b[0;36menv_tonic_compat\u001b[1;34m(env, id, parallel, sequential)\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21menv_tonic_compat\u001b[39m(env, \u001b[38;5;28mid\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m, parallel\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, sequential\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m):\n\u001b[0;32m     23\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m     24\u001b[0m \u001b[38;5;124;03m    Applies wrapper for tonic and passes random seed.\u001b[39;00m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m---> 26\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m apply_wrapper(\u001b[38;5;28;43meval\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43menv\u001b[49m\u001b[43m)\u001b[49m)\n",
      "File \u001b[1;32m<string>:1\u001b[0m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\environments\\builders.py:19\u001b[0m, in \u001b[0;36mgym_environment\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_builder\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m     17\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m gym\u001b[38;5;241m.\u001b[39mmake(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m---> 19\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m build_environment(_builder, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\environments\\builders.py:67\u001b[0m, in \u001b[0;36mbuild_environment\u001b[1;34m(builder, name, terminal_timeouts, time_feature, max_episode_steps, scaled_actions, *args, **kwargs)\u001b[0m\n\u001b[0;32m     60\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Builds and wrap an environment.\u001b[39;00m\n\u001b[0;32m     61\u001b[0m \u001b[38;5;124;03mTime limits can be properly handled with terminal_timeouts=False or\u001b[39;00m\n\u001b[0;32m     62\u001b[0m \u001b[38;5;124;03mtime_feature=True, see https://arxiv.org/pdf/1712.00378.pdf for more\u001b[39;00m\n\u001b[0;32m     63\u001b[0m \u001b[38;5;124;03mdetails.\u001b[39;00m\n\u001b[0;32m     64\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m     66\u001b[0m \u001b[38;5;66;03m# Build the environment.\u001b[39;00m\n\u001b[1;32m---> 67\u001b[0m environment \u001b[38;5;241m=\u001b[39m builder(name, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m     69\u001b[0m \u001b[38;5;66;03m# Get the default time limit.\u001b[39;00m\n\u001b[0;32m     70\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m max_episode_steps \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdefault\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\environments\\builders.py:17\u001b[0m, in \u001b[0;36mgym_environment.<locals>._builder\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_builder\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m---> 17\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m gym\u001b[38;5;241m.\u001b[39mmake(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\gym\\envs\\registration.py:156\u001b[0m, in \u001b[0;36mmake\u001b[1;34m(id, **kwargs)\u001b[0m\n\u001b[0;32m    155\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmake\u001b[39m(\u001b[38;5;28mid\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m--> 156\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m registry\u001b[38;5;241m.\u001b[39mmake(\u001b[38;5;28mid\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\gym\\envs\\registration.py:100\u001b[0m, in \u001b[0;36mEnvRegistry.make\u001b[1;34m(self, path, **kwargs)\u001b[0m\n\u001b[0;32m     98\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m     99\u001b[0m     logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mMaking new env: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m, path)\n\u001b[1;32m--> 100\u001b[0m spec \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mspec\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    101\u001b[0m env \u001b[38;5;241m=\u001b[39m spec\u001b[38;5;241m.\u001b[39mmake(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    102\u001b[0m \u001b[38;5;66;03m# We used to have people override _reset/_step rather than\u001b[39;00m\n\u001b[0;32m    103\u001b[0m \u001b[38;5;66;03m# reset/step. Set _gym_disable_underscore_compat = True on\u001b[39;00m\n\u001b[0;32m    104\u001b[0m \u001b[38;5;66;03m# your environment if you use these methods and don't want\u001b[39;00m\n\u001b[0;32m    105\u001b[0m \u001b[38;5;66;03m# compatibility code to be invoked.\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\gym\\envs\\registration.py:142\u001b[0m, in \u001b[0;36mEnvRegistry.spec\u001b[1;34m(self, path)\u001b[0m\n\u001b[0;32m    140\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m error\u001b[38;5;241m.\u001b[39mDeprecatedEnv(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mEnv \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m not found (valid versions include \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;28mid\u001b[39m, matching_envs))\n\u001b[0;32m    141\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 142\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m error\u001b[38;5;241m.\u001b[39mUnregisteredEnv(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mNo registered env with id: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;28mid\u001b[39m))\n",
      "\u001b[1;31mUnregisteredEnv\u001b[0m: No registered env with id: sconewalk_harness_h1622-v1"
     ]
    }
   ],
   "source": [
    "from main import main\n",
    "main(config,setting=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1622 fix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "설정을 바꾸지 않고 종료합니다.\n"
     ]
    }
   ],
   "source": [
    "with open(r'config\\\\scone_walk_h1622_fix.yaml','r') as y:\n",
    "    config = yaml.safe_load(y)  ## agent의 default 설정들을 딕셔너리로 받아옴\n",
    "config = configmake(config,skip_while_loop=True) ## 건들 필요 없음\n",
    "config = make_weight_dict(config,is_weight=False)  ## 건들 필요 없음\n",
    "config = make_type_dict(config,is_weight=False)  ## 건들 필요 없음, 필요시 밑에서 재지정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['name']  = \"fix_10_default\"  ## C:\\Users\\PC\\Documents\\SCONE\\trainmodel 에 저장될 이름\n",
    "epoch = 500\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'  ## trainer에서 epoch 수 지정\n",
    "  ## myutils(서재원 만듦)에서 'trainer' 에 있는 MyTrainer Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# config['tonic']['environment'] = \"deprl.environments.Gym('sconewalk_h1622_fix_noneclip-v1')\" # GYM 이름 지정은 sconegym init 에서 확인 가능\n",
    "# config[\"env_args\"]['clip_actions'] = False\n",
    "\n",
    "# config['weights']['type']['grfreduce'] = 0.005\n",
    "# config['weights']['reward']['grf_grfdelta'] = -0.7\n",
    "# config['weights']['reward']['grf_grfdelta_x'] = -0.3\n",
    "\n",
    "# config['weights']['type']['balance'] = 0.5\n",
    "# config['weights']['reward']['balance_diff_position_z'] = -0.6\n",
    "# config['weights']['reward']['balance_position_z'] = -0.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "환경을 바꾸지 않고 종료합니다.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"deprl.environments.Gym('sconewalk_h1622_fix-v1')\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = make_env(config)\n",
    "config['tonic']['environment']  ## config에서 tonic에 들어가면 0(MPO, muscle 아닌 인풋도 넣을 수 있는 모드) 또는 3으로 지정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config 파일 저장 완료\n"
     ]
    }
   ],
   "source": [
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weight가 변경되었습니다.\n",
      "type_weight를 보여줍니다.\n",
      "customreward 함수를 사용하지 않았습니다.\n",
      "reward weight를 변경하지 않았습니다.\n"
     ]
    }
   ],
   "source": [
    "from main import main\n",
    "main(config,setting=True)  # 학습 시작"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1622 Clip_Actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "설정을 바꾸지 않고 종료합니다.\n"
     ]
    }
   ],
   "source": [
    "with open(r'config\\\\scone_walk_h1622_fix_clip.yaml','r') as y:\n",
    "    config = yaml.safe_load(y)\n",
    "config = configmake(config,skip_while_loop=True)\n",
    "config = make_weight_dict(config,is_weight=False)\n",
    "config = make_type_dict(config,is_weight=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['name']  = \"clip_0.7_\"\n",
    "# config['tonic']['name'] =\"pelvis_joint_143_\"\n",
    "epoch = 600\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['environment'] = \"deprl.environments.Gym('sconewalk_h1622_fix_clip-v1',max_activation=0.7,clip_actions=True)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['env_args']['max_activation'] =0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config 파일 저장 완료\n"
     ]
    }
   ],
   "source": [
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weight가 변경되었습니다.\n",
      "type_weight를 보여줍니다.\n",
      "customreward 함수를 사용하지 않았습니다.\n",
      "reward weight를 변경하지 않았습니다.\n",
      "\u001b[1m\u001b[32mstarting new training.\u001b[0m\n",
      "Training started.\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 159.02\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 191.46\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 1, Duration: 0분 19초\n",
      "End Time: 2024-02-16 09:35:39\n",
      "------------------------------\n",
      "\u001b[1m\u001b[33mWarning: Logging new keys\u001b[0m\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 194.3\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 227.84\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 2, Duration: 1분 20초\n",
      "End Time: 2024-02-16 09:36:59\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 251.87\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 281.76\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 3, Duration: 1분 15초\n",
      "End Time: 2024-02-16 09:38:15\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 289.73\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 333.42\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 4, Duration: 1분 17초\n",
      "End Time: 2024-02-16 09:39:32\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 321.15\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 417.76\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 5, Duration: 1분 15초\n",
      "End Time: 2024-02-16 09:40:48\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 5\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 349.6\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 370.36\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 6, Duration: 1분 37초\n",
      "End Time: 2024-02-16 09:42:25\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 373.8\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 447.54\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 7, Duration: 1분 16초\n",
      "End Time: 2024-02-16 09:43:42\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 405.0\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 501.15\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 8, Duration: 1분 15초\n",
      "End Time: 2024-02-16 09:44:57\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 411.66\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 595.8\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 9, Duration: 1분 15초\n",
      "End Time: 2024-02-16 09:46:12\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 457.4\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 659.7\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 10, Duration: 1분 15초\n",
      "End Time: 2024-02-16 09:47:28\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 10\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 532.12\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 1018.53\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 11, Duration: 1분 35초\n",
      "End Time: 2024-02-16 09:49:04\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 551.25\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 2179.19\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 12, Duration: 1분 18초\n",
      "End Time: 2024-02-16 09:50:22\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 637.53\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 2352.23\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 13, Duration: 1분 18초\n",
      "End Time: 2024-02-16 09:51:41\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 749.3\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 3000.4\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 14, Duration: 1분 20초\n",
      "End Time: 2024-02-16 09:53:02\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 847.98\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5003.61\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 15, Duration: 1분 25초\n",
      "End Time: 2024-02-16 09:54:27\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 15\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1231.99\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6479.86\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 16, Duration: 1분 44초\n",
      "End Time: 2024-02-16 09:56:12\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2000.1\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6564.79\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 17, Duration: 1분 25초\n",
      "End Time: 2024-02-16 09:57:37\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2623.91\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6517.26\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 18, Duration: 1분 25초\n",
      "End Time: 2024-02-16 09:59:03\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2381.07\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6435.26\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 19, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:00:28\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3188.22\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6264.45\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 20, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:01:53\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 20\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3119.08\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6117.41\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 21, Duration: 1분 43초\n",
      "End Time: 2024-02-16 10:03:37\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2400.15\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6106.7\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 22, Duration: 1분 24초\n",
      "End Time: 2024-02-16 10:05:01\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2570.03\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6106.8\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 23, Duration: 1분 24초\n",
      "End Time: 2024-02-16 10:06:26\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2702.67\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6202.62\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 24, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:07:51\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4405.51\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6149.36\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 25, Duration: 1분 24초\n",
      "End Time: 2024-02-16 10:09:15\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 25\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3719.35\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6134.53\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 26, Duration: 1분 47초\n",
      "End Time: 2024-02-16 10:11:03\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2918.32\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5929.53\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 27, Duration: 1분 24초\n",
      "End Time: 2024-02-16 10:12:27\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2319.96\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6018.66\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 28, Duration: 1분 24초\n",
      "End Time: 2024-02-16 10:13:52\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2385.67\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5911.61\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 29, Duration: 1분 23초\n",
      "End Time: 2024-02-16 10:15:16\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2323.96\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5713.13\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 30, Duration: 1분 23초\n",
      "End Time: 2024-02-16 10:16:39\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 30\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1657.84\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5667.12\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 31, Duration: 1분 42초\n",
      "End Time: 2024-02-16 10:18:21\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2313.76\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5647.78\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 32, Duration: 1분 22초\n",
      "End Time: 2024-02-16 10:19:44\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2421.01\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5587.32\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 33, Duration: 1분 22초\n",
      "End Time: 2024-02-16 10:21:07\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2883.26\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5584.45\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 34, Duration: 1분 22초\n",
      "End Time: 2024-02-16 10:22:29\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2529.7\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5560.17\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 35, Duration: 1분 22초\n",
      "End Time: 2024-02-16 10:23:52\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 35\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2869.18\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5615.09\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 36, Duration: 1분 41초\n",
      "End Time: 2024-02-16 10:25:33\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2967.17\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5745.49\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 37, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:26:59\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3291.26\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5633.73\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 38, Duration: 1분 24초\n",
      "End Time: 2024-02-16 10:28:24\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2080.56\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5379.26\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 39, Duration: 1분 23초\n",
      "End Time: 2024-02-16 10:29:47\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2110.05\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5213.73\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 40, Duration: 1분 24초\n",
      "End Time: 2024-02-16 10:31:11\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 40\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1362.64\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5232.35\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 41, Duration: 1분 43초\n",
      "End Time: 2024-02-16 10:32:55\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 633.58\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 632.84\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 42, Duration: 1분 16초\n",
      "End Time: 2024-02-16 10:34:11\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 240.54\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 225.07\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 43, Duration: 1분 15초\n",
      "End Time: 2024-02-16 10:35:27\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 217.61\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 273.64\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 44, Duration: 1분 15초\n",
      "End Time: 2024-02-16 10:36:42\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 305.23\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 1644.08\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 45, Duration: 1분 17초\n",
      "End Time: 2024-02-16 10:37:59\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 45\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 571.99\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4606.34\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 46, Duration: 1분 41초\n",
      "End Time: 2024-02-16 10:39:41\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1095.15\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5378.51\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 47, Duration: 1분 24초\n",
      "End Time: 2024-02-16 10:41:05\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1553.74\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5172.53\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 48, Duration: 1분 24초\n",
      "End Time: 2024-02-16 10:42:30\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2770.51\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5980.22\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 49, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:43:56\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2242.0\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6110.64\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 50, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:45:21\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 50\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3507.42\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6081.37\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 51, Duration: 1분 45초\n",
      "End Time: 2024-02-16 10:47:07\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2953.08\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5663.42\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 52, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:48:33\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1276.05\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5730.51\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 53, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:49:59\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4049.14\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5372.16\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 54, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:51:24\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4992.3\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5371.92\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 55, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:52:49\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 55\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2559.12\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5300.99\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 56, Duration: 1분 43초\n",
      "End Time: 2024-02-16 10:54:33\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1454.24\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5281.56\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 57, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:55:58\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2950.78\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5306.0\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 58, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:57:23\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3085.26\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5337.07\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 59, Duration: 1분 25초\n",
      "End Time: 2024-02-16 10:58:48\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1417.36\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4876.63\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 60, Duration: 1분 24초\n",
      "End Time: 2024-02-16 11:00:13\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 60\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2933.22\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5552.83\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 61, Duration: 1분 44초\n",
      "End Time: 2024-02-16 11:01:57\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3229.99\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5529.74\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 62, Duration: 1분 24초\n",
      "End Time: 2024-02-16 11:03:22\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2152.77\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5320.2\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 63, Duration: 1분 24초\n",
      "End Time: 2024-02-16 11:04:47\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2895.39\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5258.29\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 64, Duration: 1분 24초\n",
      "End Time: 2024-02-16 11:06:11\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1831.2\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5144.02\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 65, Duration: 1분 24초\n",
      "End Time: 2024-02-16 11:07:35\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 65\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2130.77\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5117.31\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 66, Duration: 1분 43초\n",
      "End Time: 2024-02-16 11:09:19\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1892.16\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5178.91\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 67, Duration: 1분 24초\n",
      "End Time: 2024-02-16 11:10:43\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1832.48\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5171.01\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 68, Duration: 1분 23초\n",
      "End Time: 2024-02-16 11:12:07\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3780.41\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5063.0\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 69, Duration: 1분 24초\n",
      "End Time: 2024-02-16 11:13:31\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2262.98\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5134.31\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 70, Duration: 1분 23초\n",
      "End Time: 2024-02-16 11:14:55\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 70\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1946.14\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5057.84\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 71, Duration: 1분 42초\n",
      "End Time: 2024-02-16 11:16:38\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2282.31\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5059.62\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 72, Duration: 1분 23초\n",
      "End Time: 2024-02-16 11:18:01\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2449.41\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5033.48\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 73, Duration: 1분 23초\n",
      "End Time: 2024-02-16 11:19:25\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1753.37\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4965.47\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 74, Duration: 1분 23초\n",
      "End Time: 2024-02-16 11:20:49\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1161.0\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4524.59\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 75, Duration: 1분 22초\n",
      "End Time: 2024-02-16 11:22:12\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 75\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1750.18\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5161.99\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 76, Duration: 1분 43초\n",
      "End Time: 2024-02-16 11:23:55\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1437.73\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5247.72\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 77, Duration: 1분 23초\n",
      "End Time: 2024-02-16 11:25:19\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 529.23\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 615.51\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 78, Duration: 1분 16초\n",
      "End Time: 2024-02-16 11:26:35\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 232.98\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 220.16\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 79, Duration: 1분 15초\n",
      "End Time: 2024-02-16 11:27:51\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 227.43\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 371.18\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 80, Duration: 1분 15초\n",
      "End Time: 2024-02-16 11:29:06\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 80\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 367.26\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 2963.09\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 81, Duration: 1분 38초\n",
      "End Time: 2024-02-16 11:30:45\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 666.93\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5195.75\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 82, Duration: 1분 23초\n",
      "End Time: 2024-02-16 11:32:09\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1255.38\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4266.31\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 83, Duration: 1분 22초\n",
      "End Time: 2024-02-16 11:33:32\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1170.11\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5646.26\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 84, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:34:57\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1698.84\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6169.38\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 85, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:36:23\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 85\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1880.05\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6257.99\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 86, Duration: 1분 45초\n",
      "End Time: 2024-02-16 11:38:08\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4598.64\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6228.45\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 87, Duration: 1분 30초\n",
      "End Time: 2024-02-16 11:39:39\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 5478.47\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6147.14\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 88, Duration: 1분 29초\n",
      "End Time: 2024-02-16 11:41:08\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3105.17\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6052.9\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 89, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:42:34\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2909.14\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 6007.61\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 90, Duration: 1분 26초\n",
      "End Time: 2024-02-16 11:44:00\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 90\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3273.59\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5535.08\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 91, Duration: 1분 44초\n",
      "End Time: 2024-02-16 11:45:44\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1989.95\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5444.11\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 92, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:47:10\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4914.56\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5449.95\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 93, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:48:35\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3715.52\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5602.96\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 94, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:50:01\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2728.82\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5604.54\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 95, Duration: 1분 26초\n",
      "End Time: 2024-02-16 11:51:27\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 95\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3189.37\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5401.05\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 96, Duration: 1분 45초\n",
      "End Time: 2024-02-16 11:53:12\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1828.74\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5367.3\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 97, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:54:38\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1318.84\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5316.65\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 98, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:56:03\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3665.26\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5277.08\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 99, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:57:29\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4748.87\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5305.28\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 100, Duration: 1분 25초\n",
      "End Time: 2024-02-16 11:58:54\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 100\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4915.14\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5311.38\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 101, Duration: 1분 44초\n",
      "End Time: 2024-02-16 12:00:39\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3248.28\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5364.41\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 102, Duration: 1분 25초\n",
      "End Time: 2024-02-16 12:02:04\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2744.91\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5483.97\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 103, Duration: 1분 25초\n",
      "End Time: 2024-02-16 12:03:30\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4931.24\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5306.82\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 104, Duration: 1분 25초\n",
      "End Time: 2024-02-16 12:04:55\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3728.28\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4809.82\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 105, Duration: 1분 24초\n",
      "End Time: 2024-02-16 12:06:19\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 105\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2752.48\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5094.85\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 106, Duration: 1분 43초\n",
      "End Time: 2024-02-16 12:08:03\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2512.95\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5059.96\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 107, Duration: 1분 24초\n",
      "End Time: 2024-02-16 12:09:27\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2606.97\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5015.14\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 108, Duration: 1분 25초\n",
      "End Time: 2024-02-16 12:10:52\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3255.71\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5012.88\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 109, Duration: 1분 24초\n",
      "End Time: 2024-02-16 12:12:17\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2826.62\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4914.86\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 110, Duration: 1분 24초\n",
      "End Time: 2024-02-16 12:13:42\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 110\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4002.49\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5044.86\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 111, Duration: 1분 42초\n",
      "End Time: 2024-02-16 12:15:25\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3387.77\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 5026.05\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 112, Duration: 1분 24초\n",
      "End Time: 2024-02-16 12:16:50\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2516.25\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4987.99\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 113, Duration: 1분 24초\n",
      "End Time: 2024-02-16 12:18:14\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3815.11\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4944.53\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 114, Duration: 1분 24초\n",
      "End Time: 2024-02-16 12:19:38\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1867.15\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4897.32\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 115, Duration: 1분 24초\n",
      "End Time: 2024-02-16 12:21:02\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 115\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3300.15\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4823.85\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 116, Duration: 1분 42초\n",
      "End Time: 2024-02-16 12:22:45\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3283.75\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4296.69\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 117, Duration: 1분 22초\n",
      "End Time: 2024-02-16 12:24:08\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2619.67\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4665.99\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 118, Duration: 1분 22초\n",
      "End Time: 2024-02-16 12:25:31\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 4061.39\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4710.57\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 119, Duration: 1분 23초\n",
      "End Time: 2024-02-16 12:26:55\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1834.03\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4622.95\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 120, Duration: 1분 23초\n",
      "End Time: 2024-02-16 12:28:18\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 120\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2131.64\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4605.6\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 121, Duration: 1분 42초\n",
      "End Time: 2024-02-16 12:30:00\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3281.38\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4552.08\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 122, Duration: 1분 25초\n",
      "End Time: 2024-02-16 12:31:25\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2531.33\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4547.55\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 123, Duration: 1분 33초\n",
      "End Time: 2024-02-16 12:32:59\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2412.29\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4553.71\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 124, Duration: 1분 29초\n",
      "End Time: 2024-02-16 12:34:28\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 2937.67\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4639.98\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 125, Duration: 1분 33초\n",
      "End Time: 2024-02-16 12:36:01\n",
      "------------------------------\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32mSaved a checkpoint when Epochs: 125\u001b[0m\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 3162.46\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4658.08\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 126, Duration: 1분 47초\n",
      "End Time: 2024-02-16 12:37:49\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1843.75\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4607.79\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 127, Duration: 1분 30초\n",
      "End Time: 2024-02-16 12:39:19\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1939.04\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4677.94\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 128, Duration: 1분 22초\n",
      "End Time: 2024-02-16 12:40:42\n",
      "------------------------------\n",
      "\u001b[1m\u001b[32m학습   평균 reward: 1821.37\u001b[0m\n",
      "\u001b[1m\u001b[32m테스트 평균 reward: 4273.74\u001b[0m\n",
      "------------------------------\n",
      "Epoch: 129, Duration: 1분 23초\n",
      "End Time: 2024-02-16 12:42:05\n",
      "------------------------------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[7], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmain\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m main\n\u001b[1;32m----> 2\u001b[0m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43msetting\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\main.py:16\u001b[0m, in \u001b[0;36mmain\u001b[1;34m(config, setting)\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m     15\u001b[0m     set_tensor_device()\n\u001b[1;32m---> 16\u001b[0m     \u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43msetting\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43msetting\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\main.py:128\u001b[0m, in \u001b[0;36mtrain\u001b[1;34m(config, setting)\u001b[0m\n\u001b[0;32m    126\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m    127\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTraining started.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 128\u001b[0m     trainer\u001b[38;5;241m.\u001b[39mrun(config, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mtime_dict)\n\u001b[0;32m    129\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTraining finished.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    130\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\myutils\\trainer.py:70\u001b[0m, in \u001b[0;36mMyTrainer.run\u001b[1;34m(self, params, steps, epochs, episodes)\u001b[0m\n\u001b[0;32m     68\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menv_infos\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m info:\n\u001b[0;32m     69\u001b[0m     info\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menv_infos\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m---> 70\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39magent\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39minfo, steps\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msteps)\n\u001b[0;32m     72\u001b[0m scores \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m info[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrewards\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m     73\u001b[0m lengths \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\custom_agents.py:101\u001b[0m, in \u001b[0;36mdep_factory.<locals>.DetSwitchDep.update\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    100\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupdate\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m--> 101\u001b[0m     \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\custom_agents.py:60\u001b[0m, in \u001b[0;36mdep_factory.<locals>.InitExploreDEP.update\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m     59\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupdate\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m---> 60\u001b[0m     \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\torch\\agents\\mpo.py:82\u001b[0m, in \u001b[0;36mMPO.update\u001b[1;34m(self, observations, rewards, resets, terminations, steps)\u001b[0m\n\u001b[0;32m     80\u001b[0m \u001b[38;5;66;03m# Update the model if the replay is ready.\u001b[39;00m\n\u001b[0;32m     81\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreplay\u001b[38;5;241m.\u001b[39mready(steps):\n\u001b[1;32m---> 82\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_update\u001b[49m\u001b[43m(\u001b[49m\u001b[43msteps\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\torch\\agents\\mpo.py:106\u001b[0m, in \u001b[0;36mMPO._update\u001b[1;34m(self, steps)\u001b[0m\n\u001b[0;32m    104\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreplay\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;241m*\u001b[39mkeys, steps\u001b[38;5;241m=\u001b[39msteps):\n\u001b[0;32m    105\u001b[0m     batch \u001b[38;5;241m=\u001b[39m {k: torch\u001b[38;5;241m.\u001b[39mas_tensor(v) \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m batch\u001b[38;5;241m.\u001b[39mitems()}\n\u001b[1;32m--> 106\u001b[0m     infos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_update_actor_critic(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mbatch)\n\u001b[0;32m    108\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m infos:\n\u001b[0;32m    109\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m infos[key]\u001b[38;5;241m.\u001b[39mitems():\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\torch\\agents\\mpo.py:121\u001b[0m, in \u001b[0;36mMPO._update_actor_critic\u001b[1;34m(self, observations, actions, next_observations, rewards, discounts)\u001b[0m\n\u001b[0;32m    118\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_update_actor_critic\u001b[39m(\n\u001b[0;32m    119\u001b[0m     \u001b[38;5;28mself\u001b[39m, observations, actions, next_observations, rewards, discounts\n\u001b[0;32m    120\u001b[0m ):\n\u001b[1;32m--> 121\u001b[0m     critic_infos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcritic_updater\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    122\u001b[0m \u001b[43m        \u001b[49m\u001b[43mobservations\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mactions\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnext_observations\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrewards\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdiscounts\u001b[49m\n\u001b[0;32m    123\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    124\u001b[0m     actor_infos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mactor_updater(observations)\n\u001b[0;32m    125\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mupdate_targets()\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\torch\\updaters\\critics.py:300\u001b[0m, in \u001b[0;36mExpectedSARSA.__call__\u001b[1;34m(self, observations, actions, next_observations, rewards, discounts)\u001b[0m\n\u001b[0;32m    297\u001b[0m     returns \u001b[38;5;241m=\u001b[39m rewards \u001b[38;5;241m+\u001b[39m discounts \u001b[38;5;241m*\u001b[39m next_values\n\u001b[0;32m    299\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[1;32m--> 300\u001b[0m values \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcritic\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobservations\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mactions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    301\u001b[0m loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mloss(returns, values)\n\u001b[0;32m    303\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\torch\\nn\\modules\\module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\torch\\nn\\modules\\module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\torch\\models\\critics.py:95\u001b[0m, in \u001b[0;36mCritic.forward\u001b[1;34m(self, *inputs)\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39minputs):\n\u001b[1;32m---> 95\u001b[0m     out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     96\u001b[0m     out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtorso(out)\n\u001b[0;32m     97\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhead(out)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\torch\\nn\\modules\\module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\torch\\nn\\modules\\module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\torch\\models\\encoders.py:32\u001b[0m, in \u001b[0;36mObservationActionEncoder.forward\u001b[1;34m(self, observations, actions)\u001b[0m\n\u001b[0;32m     30\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, observations, actions):\n\u001b[0;32m     31\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobservation_normalizer:\n\u001b[1;32m---> 32\u001b[0m         observations \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mobservation_normalizer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobservations\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     33\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcat([observations, actions], dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\torch\\nn\\modules\\module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\torch\\nn\\modules\\module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\deprl\\vendor\\tonic\\torch\\normalizers\\mean_stds.py:39\u001b[0m, in \u001b[0;36mMeanStd.forward\u001b[1;34m(self, val)\u001b[0m\n\u001b[0;32m     37\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, val):\n\u001b[0;32m     38\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[1;32m---> 39\u001b[0m         val \u001b[38;5;241m=\u001b[39m (\u001b[43mval\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_mean\u001b[49m) \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_std\n\u001b[0;32m     40\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclip \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m     41\u001b[0m             val \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mclamp(val, \u001b[38;5;241m-\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclip, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclip)\n",
      "File \u001b[1;32mc:\\Users\\PC\\Desktop\\SJW\\dep\\pythongait\\lib\\site-packages\\torch\\utils\\_device.py:77\u001b[0m, in \u001b[0;36mDeviceContext.__torch_function__\u001b[1;34m(self, func, types, args, kwargs)\u001b[0m\n\u001b[0;32m     75\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m func \u001b[38;5;129;01min\u001b[39;00m _device_constructors() \u001b[38;5;129;01mand\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdevice\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m     76\u001b[0m     kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdevice\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice\n\u001b[1;32m---> 77\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from main import main\n",
    "main(config,setting=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1922 ACt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(r'config\\scone_walk_h1922_actuator.yaml','r') as f:\n",
    "    config = yaml.safe_load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "설정을 바꾸지 않고 종료합니다.\n"
     ]
    }
   ],
   "source": [
    "config = configmake(config,skip_while_loop=True)\n",
    "config = make_weight_dict(config,is_weight=False)\n",
    "config = make_type_dict(config,is_weight=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['name']  = \"actuator_None_default\"\n",
    "config['tonic']['name']  = \"actuator_default_v2_\"\n",
    "config['tonic']['name'] = 'actuator_default_v3_'\n",
    "config['tonic']['name'] = 'actuator_script'\n",
    "config['tonic']['name'] = 'actuator_default_20_'\n",
    "epoch = 500\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['environment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = make_env(config)\n",
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from main import main\n",
    "main(config,setting=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1922 Motor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(r'config\\\\scone_walk_h1922_motor.yaml','r') as y:\n",
    "    config = yaml.safe_load(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "설정을 바꾸지 않고 종료합니다.\n"
     ]
    }
   ],
   "source": [
    "config = configmake(config,skip_while_loop=True)\n",
    "config = make_weight_dict(config,is_weight=False)\n",
    "config = make_type_dict(config,is_weight=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 600\n",
    "config['tonic']['trainer'] = f'MyTrainer(steps=int(2e5)*{epoch}, epoch_steps=int(2e5), save_steps=int(1e6))'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['weights']['type']['trunkbalance'] = 10\n",
    "config['weights']['reward']['trunkbalance_position_z'] = -0.5\n",
    "config['weights']['reward']['trunkbalance_com_z'] = -0.5\n",
    "config = make_env(config)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "config['tonic']['name']  = \"motor_balance\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config 파일 저장 완료\n"
     ]
    }
   ],
   "source": [
    "with open(f\"trainconfig/{config['tonic']['name']}.yaml\",'w') as y:\n",
    "    yaml.dump(config,y)\n",
    "    print(\"config 파일 저장 완료\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"myutils.environments.Gym('sconewalk_h1922_motor-v1', rwd_type_weights={'trunkbalance': 10},rwd_weights={'trunkbalance_position_z': -0.5, 'trunkbalance_com_z': -0.5})\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config['tonic']['environment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'config' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmain\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m main\n\u001b[1;32m----> 2\u001b[0m main(\u001b[43mconfig\u001b[49m,setting\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'config' is not defined"
     ]
    }
   ],
   "source": [
    "from main import main\n",
    "main(config,setting=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python_scone",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}