DoubleDeepQNetwork.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import cv2  #opencv\n",
    "from IPython.display import clear_output\n",
    "from random import randint\n",
    "import io\n",
    "from io import BytesIO\n",
    "import time\n",
    "import os\n",
    "\n",
    "#keras imports\n",
    "from keras.models import model_from_json\n",
    "from keras.models import Sequential\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.optimizers import SGD , Adam\n",
    "from keras.layers.core import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D\n",
    "\n",
    "\n",
    "from collections import deque\n",
    "import random\n",
    "import pickle\n",
    "import _pickle as cPickle\n",
    "import base64\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path variables\n",
    "game_url = \"chrome://dino\"\n",
    "chromedriver_path = \"./chrome_driver/chromedriver\"\n",
    "loss_file_path = \"./objects_ddqn/loss_df.csv\"\n",
    "actions_file_path = \"./objects_ddqn/actions_df.csv\"\n",
    "q_value_file_path = \"./objects_ddqn/q_values.csv\"\n",
    "scores_file_path = \"./objects_ddqn/scores_df.csv\"\n",
    "\n",
    "#scripts\n",
    "#create id for canvas for faster selection from DOM\n",
    "init_script = \"document.getElementsByClassName('runner-canvas')[0].id = 'runner-canvas'\"\n",
    "\n",
    "#get image from canvas\n",
    "getbase64Script = \"canvasRunner = document.getElementById('runner-canvas'); \\\n",
    "return canvasRunner.toDataURL().substring(22)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "* Game class: Selenium interfacing between the python and browser\n",
    "* __init__():  Launch the broswer window using the attributes in chrome_options\n",
    "* crashed() : return true if the agent as crashed on an obstacles. Gets javascript variable from game decribing the state\n",
    "* get_playing(): true if game in progress, false is crashed or paused\n",
    "* restart() : sends a signal to browser-javascript to restart the game\n",
    "* press_up(): sends a single to press up get to the browser\n",
    "* get_score(): gets current game score from javascript variables.\n",
    "* pause(): pause the game\n",
    "* resume(): resume a paused game if not crashed\n",
    "* end(): close the browser and end the game\n",
    "'''\n",
    "class Game:\n",
    "    def __init__(self,custom_config=True):\n",
    "        chrome_options = Options()\n",
    "        chrome_options.add_argument(\"disable-infobars\")\n",
    "        chrome_options.add_argument(\"--mute-audio\")\n",
    "        self._driver = webdriver.Chrome(executable_path = chromedriver_path,chrome_options=chrome_options)\n",
    "        self._driver.set_window_position(x=-10,y=0)\n",
    "        self._driver.get('chrome://dino')\n",
    "        self._driver.execute_script(\"Runner.config.ACCELERATION=0\")\n",
    "        self._driver.execute_script(init_script)\n",
    "    def crashed(self):\n",
    "        return self._driver.execute_script(\"return Runner.instance_.crashed\")\n",
    "    def get_playing(self):\n",
    "        return self._driver.execute_script(\"return Runner.instance_.playing\")\n",
    "    def restart(self):\n",
    "        self._driver.execute_script(\"Runner.instance_.restart()\")\n",
    "    def press_up(self):\n",
    "        self._driver.find_element_by_tag_name(\"body\").send_keys(Keys.ARROW_UP)\n",
    "    def get_score(self):\n",
    "        score_array = self._driver.execute_script(\"return Runner.instance_.distanceMeter.digits\")\n",
    "        score = ''.join(score_array) # the javascript object is of type array with score in the formate[1,0,0] which is 100.\n",
    "        return int(score)\n",
    "    def pause(self):\n",
    "        return self._driver.execute_script(\"return Runner.instance_.stop()\")\n",
    "    def resume(self):\n",
    "        return self._driver.execute_script(\"return Runner.instance_.play()\")\n",
    "    def end(self):\n",
    "        self._driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dino_Agent:\n",
    "    def __init__(self,game): #takes game as input for taking actions\n",
    "        self._game = game; \n",
    "        self.jump(); #to start the game, we need to jump once\n",
    "    def running(self):\n",
    "        return self._game.get_playing()\n",
    "    def is_crashed(self):\n",
    "        return self._game.crashed()\n",
    "    def jump(self):\n",
    "        self._game.press_up()\n",
    "    def duck(self):\n",
    "        self._game.press_down()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Game_state:\n",
    "    def __init__(self,agent,game):\n",
    "        self._agent = agent\n",
    "        self._game = game\n",
    "        self._display = show_img() #display the processed image on screen using openCV, implemented using python coroutine \n",
    "        self._display.__next__() # initiliaze the display coroutine \n",
    "    def get_state(self,actions):\n",
    "        actions_df.loc[len(actions_df)] = actions[1] # storing actions in a dataframe\n",
    "        score = self._game.get_score() \n",
    "        reward = 0.1\n",
    "        game_over = False #game over\n",
    "        if actions[1] == 1:\n",
    "            self._agent.jump()\n",
    "        image = grab_screen(self._game._driver) \n",
    "        self._display.send(image) #display the image on screen\n",
    "        if self._agent.is_crashed():\n",
    "            scores_df.loc[len(loss_df)] = score # log the score when game is over\n",
    "            self._game.restart()\n",
    "            reward = -1\n",
    "            game_over = True\n",
    "        return image, reward, game_over #return the Experience tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_object(obj, name):\n",
    "    with open('objects_ddqn/'+ name + '.pkl', 'wb') as f: #dump files into objects folder\n",
    "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n",
    "def load_object(name):\n",
    "    with open('objects_ddqn/' + name + '.pkl', 'rb') as f:\n",
    "        return pickle.load(f)\n",
    "\n",
    "def grab_screen(_driver):\n",
    "    image_b64 = _driver.execute_script(getbase64Script)\n",
    "    screen = np.array(Image.open(BytesIO(base64.b64decode(image_b64))))\n",
    "    image = img_process(screen)#processing image as required\n",
    "    return image\n",
    "\n",
    "def img_process(image):\n",
    "    \n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) #RGB to Grey Scale\n",
    "    image = image[:300, :500] #Crop Region of Interest(ROI)\n",
    "    image = cv2.resize(image, (80,80))\n",
    "    return  image\n",
    "\n",
    "def show_img(graphs = False):\n",
    "    \"\"\"\n",
    "    Show images in new window\n",
    "    \"\"\"\n",
    "    while True:\n",
    "        screen = (yield)\n",
    "        window_title = \"logs\" if graphs else \"game_play\"\n",
    "        cv2.namedWindow(window_title, cv2.WINDOW_NORMAL)        \n",
    "        imS = cv2.resize(screen, (800, 400)) \n",
    "        cv2.imshow(window_title, screen)\n",
    "        if (cv2.waitKey(1) & 0xFF == ord('q')):\n",
    "            cv2.destroyAllWindows()\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Intialize log structures from file if exists else create new\n",
    "loss_df = pd.read_csv(loss_file_path) if os.path.isfile(loss_file_path) else pd.DataFrame(columns =['loss'])\n",
    "scores_df = pd.read_csv(scores_file_path) if os.path.isfile(loss_file_path) else pd.DataFrame(columns = ['scores'])\n",
    "actions_df = pd.read_csv(actions_file_path) if os.path.isfile(actions_file_path) else pd.DataFrame(columns = ['actions'])\n",
    "q_values_df =pd.read_csv(actions_file_path) if os.path.isfile(q_value_file_path) else pd.DataFrame(columns = ['qvalues'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#game parameters\n",
    "ACTIONS = 2 # possible actions: jump, do nothing\n",
    "GAMMA = 0.99 # decay rate of past observations original 0.99\n",
    "OBSERVATION = 100 # timesteps to observe before training\n",
    "EXPLORE = 100000  # frames over which to anneal epsilon\n",
    "FINAL_EPSILON = 0.0001 # final value of epsilon\n",
    "INITIAL_EPSILON = 0.1 # starting value of epsilon\n",
    "REPLAY_MEMORY = 50000 # number of previous transitions to remember\n",
    "BATCH_SIZE = 16 # size of minibatch\n",
    "FRAME_PER_ACTION = 1\n",
    "LEARNING_RATE = 1e-4\n",
    "img_rows , img_cols = 80,80\n",
    "img_channels = 4 #We stack 4 frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training variables saved as checkpoints to filesystem to resume training from the same step\n",
    "def init_cache():\n",
    "    \"\"\"initial variable caching, done only once\"\"\"\n",
    "    save_object(INITIAL_EPSILON,\"epsilon\")\n",
    "    t = 0\n",
    "    save_object(t,\"time\")\n",
    "    D = deque()\n",
    "    save_object(D,\"D\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Call only once to init file structure\n",
    "'''\n",
    "init_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    print(\"Now we build the model\")\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, (8, 8), padding='same',strides=(4, 4),input_shape=(img_cols,img_rows,img_channels)))  #80*80*4\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(64, (4, 4),strides=(2, 2),  padding='same'))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(64, (3, 3),strides=(1, 1),  padding='same'))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(512))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dense(ACTIONS))\n",
    "    adam = Adam(lr=LEARNING_RATE)\n",
    "    model.compile(loss='mse',optimizer=adam)\n",
    "    \n",
    "    #create model file if not present\n",
    "    if not os.path.isfile(loss_file_path):\n",
    "        model.save_weights('model_ddqn.h5')\n",
    "    print(\"We finish building the model\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' \n",
    "main training module\n",
    "Parameters:\n",
    "* model => Keras Model to be trained\n",
    "* game_state => Game State module with access to game environment and dino\n",
    "* observe => flag to indicate whether the model is to be trained(weight updates), else just play\n",
    "'''\n",
    "def train_Network(q_eval,q_target,game_state,observe=False):\n",
    "    last_time = time.time()\n",
    "    # store the previous observations in replay memory\n",
    "    D = load_object(\"D\") #load from file system\n",
    "    # get the first state by doing nothing\n",
    "    do_nothing = np.zeros(ACTIONS)\n",
    "    do_nothing[0] =1 #0 => do nothing,\n",
    "                     #1=> jump\n",
    "    \n",
    "    x_t, r_0, terminal = game_state.get_state(do_nothing) # get next step after performing the action\n",
    "    \n",
    "\n",
    "    s_t = np.stack((x_t, x_t, x_t, x_t), axis=2) # stack 4 images to create placeholder input\n",
    "    \n",
    "\n",
    "    \n",
    "    s_t = s_t.reshape(1, s_t.shape[0], s_t.shape[1], s_t.shape[2])  #1*20*40*4\n",
    "    \n",
    "    initial_state = s_t \n",
    "\n",
    "    if observe :\n",
    "        OBSERVE = 999999999    #We keep observe, never train\n",
    "        epsilon = FINAL_EPSILON\n",
    "        print (\"Now we load weight\")\n",
    "        q_eval.load_weights(\"model_ddqn.h5\")\n",
    "        adam = Adam(lr=LEARNING_RATE)\n",
    "        q_eval.compile(loss='mse',optimizer=adam)\n",
    "        print (\"Weight load successfully\")    \n",
    "    else:                       #We go to training mode\n",
    "        OBSERVE = OBSERVATION\n",
    "        epsilon = load_object(\"epsilon\") \n",
    "        q_eval.load_weights(\"model_ddqn.h5\")\n",
    "        adam = Adam(lr=LEARNING_RATE)\n",
    "        q_eval.compile(loss='mse',optimizer=adam)\n",
    "\n",
    "    t = load_object(\"time\") # resume from the previous time step stored in file system\n",
    "    while (True): #endless running\n",
    "        \n",
    "        loss = 0\n",
    "        Q_sa = 0\n",
    "\n",
    "        action_index = 0\n",
    "        r_t = 0 #reward at 4\n",
    "        a_t = np.zeros([ACTIONS]) # action at t\n",
    "        \n",
    "        #choose an action epsilon greedy\n",
    "        if t % FRAME_PER_ACTION == 0: #parameter to skip frames for actions\n",
    "            if  random.random() <= epsilon: #randomly explore an action\n",
    "                print(\"----------Random Action----------\")\n",
    "                action_index = random.randrange(ACTIONS)\n",
    "                a_t[action_index] = 1\n",
    "            else: # predict the output\n",
    "                q = q_target.predict(s_t)       #input a stack of 4 images, get the prediction\n",
    "                max_Q = np.argmax(q)         # chosing index with maximum q value\n",
    "                action_index = max_Q \n",
    "                a_t[action_index] = 1        # o=> do nothing, 1=> jump\n",
    "                \n",
    "        \n",
    "        #run the selected action and observed next state and reward\n",
    "        x_t1, r_t, terminal = game_state.get_state(a_t)\n",
    "        print('fps: {0}'.format(1 / (time.time()-last_time))) # helpful for measuring frame rate\n",
    "        last_time = time.time()\n",
    "        x_t1 = x_t1.reshape(1, x_t1.shape[0], x_t1.shape[1], 1) #1x20x40x1\n",
    "        s_t1 = np.append(x_t1, s_t[:, :, :, :3], axis=3) # append the new image to input stack and remove the first one\n",
    "        \n",
    "        \n",
    "        # store the transition in D\n",
    "        D.append((s_t, action_index, r_t, s_t1, terminal))\n",
    "        if len(D) > REPLAY_MEMORY:\n",
    "            D.popleft()\n",
    "\n",
    "        #only train if done observing\n",
    "        if t > OBSERVE: \n",
    "            \n",
    "            #sample a minibatch to train on\n",
    "            minibatch = random.sample(D, BATCH_SIZE)\n",
    "            inputs = np.zeros((BATCH_SIZE, s_t.shape[1], s_t.shape[2], s_t.shape[3]))   #32, 20, 40, 4\n",
    "            targets = np.zeros((inputs.shape[0], ACTIONS))                         #32, 2\n",
    "\n",
    "            #Now we do the experience replay\n",
    "            for i in range(0, len(minibatch)):\n",
    "                state_t = minibatch[i][0]    # 4D stack of images\n",
    "                action_t = minibatch[i][1]   #This is action index\n",
    "                reward_t = minibatch[i][2]   #reward at state_t due to action_t\n",
    "                state_t1 = minibatch[i][3]   #next state\n",
    "                terminal = minibatch[i][4]   #wheather the agent died or survided due the action\n",
    "                \n",
    "\n",
    "                inputs[i:i + 1] = state_t    \n",
    "                \n",
    "                #q_next = q_target.predict(state_t1)\n",
    "                #q_eval = q_eval.predict(state_t1)\n",
    "                #q_pred = q_eval.predict(state_t)\n",
    "                \n",
    "                #targets[i] = q_pred\n",
    "                Q_sa = q_target.predict(state_t1)      #predict q values for next step\n",
    "                #old_q = q_eval.predict(state_t1)\n",
    "                targets[i] = q_target.predict(state_t)  # predicted q values\n",
    "                #max_Actions = np.argmax(old_q)\n",
    "                q_eval = q_target\n",
    "                #q_target  = targets[i]\n",
    "\n",
    "                if terminal:\n",
    "                    targets[i, action_t] = reward_t # if terminated, only equals reward\n",
    "                else:\n",
    "                    #targets[i, action_t] = reward_t + GAMMA * np.max(Q_sa)\n",
    "                    targets[i, action_t] = reward_t + GAMMA * np.max(Q_sa)\n",
    "\n",
    "            loss += q_target.train_on_batch(inputs, targets)\n",
    "            loss_df.loc[len(loss_df)] = loss\n",
    "            q_values_df.loc[len(q_values_df)] = np.max(Q_sa)\n",
    "        s_t = initial_state if terminal else s_t1 #reset game to initial frame if terminate\n",
    "        t = t + 1\n",
    "        \n",
    "        #We reduced the epsilon (exploration parameter) gradually\n",
    "        if epsilon > FINAL_EPSILON and t > OBSERVE:\n",
    "            epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / EXPLORE \n",
    "            q_target.set_weights(q_eval.get_weights())\n",
    "\n",
    "        # save progress every 1000 iterations\n",
    "        if t % 1000 == 0:\n",
    "            print(\"Now we save model\")\n",
    "            game_state._game.pause() #pause game while saving to filesystem\n",
    "            q_target.save_weights(\"model_ddqn.h5\", overwrite=True)\n",
    "            save_object(D,\"D\") #saving episodes\n",
    "            save_object(t,\"time\") #caching time steps\n",
    "            save_object(epsilon,\"epsilon\") #cache epsilon to avoid repeated randomness in actions\n",
    "            loss_df.to_csv(\"./objects_ddqn/loss_df.csv\",index=False)\n",
    "            scores_df.to_csv(\"./objects_ddqn/scores_df.csv\",index=False)\n",
    "            actions_df.to_csv(\"./objects_ddqn/actions_df.csv\",index=False)\n",
    "            q_values_df.to_csv(q_value_file_path,index=False)\n",
    "            with open(\"model_ddqn.json\", \"w\") as outfile:\n",
    "                json.dump(q_target.to_json(), outfile)\n",
    "            clear_output()\n",
    "            game_state._game.resume()\n",
    "        # print info\n",
    "        state = \"\"\n",
    "        if t <= OBSERVE:\n",
    "            state = \"observe\"\n",
    "        elif t > OBSERVE and t <= OBSERVE + EXPLORE:\n",
    "            state = \"explore\"\n",
    "        else:\n",
    "            state = \"train\"\n",
    "\n",
    "        print(\"TIMESTEP\", t, \"/ STATE\", state, \"/ EPSILON\", epsilon, \"/ ACTION\", action_index, \"/ REWARD\", r_t, \"/ Q_MAX \", np.max(Q_sa), \"/ Loss \", loss)\n",
    "\n",
    "    print(\"Episode finished!\")\n",
    "    print(\"************************\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#main function\n",
    "def play_Game(observe=False):\n",
    "    game = Game()\n",
    "    dino = Dino_Agent(game)\n",
    "    game_state = Game_state(dino,game)    \n",
    "    #model = build_model()\n",
    "    q_eval = build_model()\n",
    "    q_target = build_model()\n",
    "    try:\n",
    "        train_Network(q_eval,q_target,game_state,observe=observe)\n",
    "    except StopIteration:\n",
    "        game.end()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIMESTEP 260000 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.490399 / Loss  0.018630996346473694\n",
      "fps: 0.18478584348733726\n",
      "TIMESTEP 260001 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.686392 / Loss  0.03000415861606598\n",
      "fps: 8.280187228430925\n",
      "TIMESTEP 260002 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.112508 / Loss  0.022375309839844704\n",
      "fps: 7.155232033463724\n",
      "TIMESTEP 260003 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.83216 / Loss  0.007487312890589237\n",
      "fps: 7.19832222943032\n",
      "TIMESTEP 260004 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.533012 / Loss  0.04205619543790817\n",
      "fps: 7.041262433373904\n",
      "TIMESTEP 260005 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.766775 / Loss  0.007928796112537384\n",
      "fps: 7.640399006127907\n",
      "TIMESTEP 260006 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.696276 / Loss  0.06255471706390381\n",
      "fps: 8.607178695801174\n",
      "TIMESTEP 260007 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.585823 / Loss  0.020926877856254578\n",
      "fps: 8.962915792307923\n",
      "TIMESTEP 260008 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.753661 / Loss  0.042920611798763275\n",
      "fps: 8.169008707913859\n",
      "TIMESTEP 260009 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.173439 / Loss  0.020721381530165672\n",
      "fps: 8.994630204669836\n",
      "TIMESTEP 260010 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.630689 / Loss  0.007198752835392952\n",
      "fps: 8.709591276919594\n",
      "TIMESTEP 260011 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.989881 / Loss  0.030273541808128357\n",
      "fps: 8.066279472287397\n",
      "TIMESTEP 260012 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.526656 / Loss  0.015339819714426994\n",
      "fps: 8.838598738999966\n",
      "TIMESTEP 260013 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.922983 / Loss  0.022688785567879677\n",
      "fps: 8.929674859060174\n",
      "TIMESTEP 260014 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.851372 / Loss  0.147363543510437\n",
      "fps: 8.234652928842364\n",
      "TIMESTEP 260015 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.591733 / Loss  0.04342055320739746\n",
      "fps: 9.223419718743472\n",
      "TIMESTEP 260016 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.768241 / Loss  0.01619860529899597\n",
      "fps: 7.776734527385323\n",
      "TIMESTEP 260017 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.726687 / Loss  0.046237267553806305\n",
      "fps: 7.203935255159526\n",
      "TIMESTEP 260018 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.422272 / Loss  0.02161077968776226\n",
      "fps: 7.38089447704856\n",
      "TIMESTEP 260019 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.784227 / Loss  0.02233283966779709\n",
      "fps: 7.370842120061859\n",
      "TIMESTEP 260020 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.014163 / Loss  0.01568804681301117\n",
      "fps: 6.173804887167689\n",
      "TIMESTEP 260021 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.400119 / Loss  0.02946113608777523\n",
      "fps: 7.079340592197761\n",
      "TIMESTEP 260022 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.708705 / Loss  0.025475289672613144\n",
      "fps: 7.027494713843865\n",
      "TIMESTEP 260023 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.899663 / Loss  0.014720072969794273\n",
      "fps: 7.128805493235434\n",
      "TIMESTEP 260024 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.62402 / Loss  0.0073502580635249615\n",
      "fps: 8.7385338494681\n",
      "TIMESTEP 260025 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.683535 / Loss  0.012966628186404705\n",
      "fps: 7.103138770436556\n",
      "TIMESTEP 260026 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD -1 / Q_MAX  11.541252 / Loss  0.022465042769908905\n",
      "fps: 8.339587902310221\n",
      "TIMESTEP 260027 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.557812 / Loss  0.009892359375953674\n",
      "fps: 8.57618634779906\n",
      "TIMESTEP 260028 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.5999565 / Loss  0.014427460730075836\n",
      "fps: 8.439600063584175\n",
      "TIMESTEP 260029 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.508799 / Loss  0.010229755192995071\n",
      "fps: 8.363217272361112\n",
      "TIMESTEP 260030 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.322428 / Loss  0.05742413178086281\n",
      "fps: 6.178907205256257\n",
      "TIMESTEP 260031 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.303426 / Loss  0.019531110301613808\n",
      "fps: 8.49942348337626\n",
      "TIMESTEP 260032 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.636919 / Loss  0.0356350801885128\n",
      "fps: 8.451742516598994\n",
      "TIMESTEP 260033 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.334439 / Loss  0.010969233699142933\n",
      "fps: 8.646783666720955\n",
      "TIMESTEP 260034 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.86125 / Loss  0.016490675508975983\n",
      "fps: 8.661228838934296\n",
      "TIMESTEP 260035 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.558379 / Loss  0.013049780391156673\n",
      "fps: 8.375709407512431\n",
      "TIMESTEP 260036 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.667191 / Loss  0.018500277772545815\n",
      "fps: 8.676045383557252\n",
      "TIMESTEP 260037 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.260223 / Loss  0.004582617431879044\n",
      "fps: 8.59696401998844\n",
      "TIMESTEP 260038 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.761116 / Loss  0.1649337112903595\n",
      "fps: 8.355603366701528\n",
      "TIMESTEP 260039 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.399732 / Loss  0.021082405000925064\n",
      "fps: 8.630219628725808\n",
      "TIMESTEP 260040 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.709681 / Loss  0.02066943235695362\n",
      "fps: 8.637132860940708\n",
      "TIMESTEP 260041 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.594196 / Loss  0.0371568500995636\n",
      "fps: 7.972370534399027\n",
      "TIMESTEP 260042 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.607465 / Loss  0.06432416290044785\n",
      "fps: 8.663178788523462\n",
      "TIMESTEP 260043 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.602029 / Loss  0.011498219333589077\n",
      "fps: 8.883922198240713\n",
      "TIMESTEP 260044 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.417744 / Loss  0.019598405808210373\n",
      "fps: 8.540735582189967\n",
      "TIMESTEP 260045 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.539327 / Loss  0.01373974047601223\n",
      "fps: 9.022395171195821\n",
      "TIMESTEP 260046 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.441315 / Loss  0.011629585176706314\n",
      "fps: 8.624860169770349\n",
      "TIMESTEP 260047 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.921282 / Loss  0.0053969621658325195\n",
      "fps: 8.515990181129332\n",
      "TIMESTEP 260048 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.515016 / Loss  0.007834997959434986\n",
      "fps: 8.925133473634036\n",
      "TIMESTEP 260049 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.619988 / Loss  0.005505772307515144\n",
      "fps: 8.851338261642127\n",
      "TIMESTEP 260050 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.485142 / Loss  0.019975654780864716\n",
      "fps: 8.253481992813654\n",
      "TIMESTEP 260051 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.111002 / Loss  0.015034309588372707\n",
      "fps: 8.68824830763367\n",
      "TIMESTEP 260052 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.527177 / Loss  0.02380475401878357\n",
      "fps: 8.491129902442086\n",
      "TIMESTEP 260053 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.290929 / Loss  0.017030945047736168\n",
      "fps: 7.814813158640016\n",
      "TIMESTEP 260054 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.394065 / Loss  0.014399321749806404\n",
      "fps: 8.252994789617013\n",
      "TIMESTEP 260055 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.5383 / Loss  0.08786793053150177\n",
      "fps: 8.33541140441701\n",
      "TIMESTEP 260056 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.611298 / Loss  0.023135725408792496\n",
      "fps: 7.782954079445695\n",
      "TIMESTEP 260057 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.322127 / Loss  0.015992190688848495\n",
      "fps: 8.721580500342059\n",
      "TIMESTEP 260058 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.583774 / Loss  0.017231790348887444\n",
      "fps: 8.186052321360261\n",
      "TIMESTEP 260059 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.250692 / Loss  0.013352401554584503\n",
      "fps: 7.4015910326182865\n",
      "TIMESTEP 260060 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.413726 / Loss  0.011324891820549965\n",
      "fps: 8.827772001532225\n",
      "TIMESTEP 260061 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  -1.6324617 / Loss  3.73258638381958\n",
      "fps: 8.848238602442477\n",
      "TIMESTEP 260062 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.54243 / Loss  0.01844879984855652\n",
      "fps: 7.665995894950204\n",
      "TIMESTEP 260063 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.459646 / Loss  0.015836438164114952\n",
      "fps: 7.50446227286065\n",
      "TIMESTEP 260064 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.620057 / Loss  0.02407626062631607\n",
      "fps: 7.419318547659564\n",
      "TIMESTEP 260065 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.523406 / Loss  0.016240034252405167\n",
      "fps: 6.849543070280298\n",
      "TIMESTEP 260066 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.507832 / Loss  0.0458708219230175\n",
      "fps: 5.255967358723221\n",
      "TIMESTEP 260067 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.151594 / Loss  0.012148914858698845\n",
      "fps: 8.870919915823313\n",
      "TIMESTEP 260068 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.411277 / Loss  0.024572113528847694\n",
      "fps: 7.312347452544666\n",
      "TIMESTEP 260069 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.346846 / Loss  0.04576551169157028\n",
      "fps: 7.677614803506479\n",
      "TIMESTEP 260070 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.354394 / Loss  0.6582013368606567\n",
      "fps: 7.3750672570044\n",
      "TIMESTEP 260071 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.695617 / Loss  0.06052529066801071\n",
      "fps: 7.396500583706306\n",
      "TIMESTEP 260072 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.286533 / Loss  0.022450225427746773\n",
      "fps: 7.554065103613225\n",
      "TIMESTEP 260073 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.40419 / Loss  0.06631738692522049\n",
      "fps: 8.669499793303018\n",
      "TIMESTEP 260074 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.6487465 / Loss  0.006447712890803814\n",
      "fps: 8.573311421259618\n",
      "TIMESTEP 260075 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.503094 / Loss  0.01953347958624363\n",
      "fps: 8.75564462257849\n",
      "TIMESTEP 260076 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.346819 / Loss  0.024805579334497452\n",
      "fps: 9.06715782282925\n",
      "TIMESTEP 260077 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.180846 / Loss  0.022504225373268127\n",
      "fps: 8.49942348337626\n",
      "TIMESTEP 260078 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.4651375 / Loss  0.013058525510132313\n",
      "fps: 7.8331733444889755\n",
      "TIMESTEP 260079 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.497188 / Loss  0.01180961262434721\n",
      "fps: 7.362251274345976\n",
      "TIMESTEP 260080 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.246872 / Loss  0.010879579000175\n",
      "fps: 7.901883579063975\n",
      "TIMESTEP 260081 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.518488 / Loss  0.029736243188381195\n",
      "fps: 7.008893330169478\n",
      "TIMESTEP 260082 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.443345 / Loss  0.008966583758592606\n",
      "fps: 8.171189003007964\n",
      "TIMESTEP 260083 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.629211 / Loss  0.008030232973396778\n",
      "fps: 7.950126711361564\n",
      "TIMESTEP 260084 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.740176 / Loss  0.0211342740803957\n",
      "fps: 8.937019249208426\n",
      "TIMESTEP 260085 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.458045 / Loss  0.01313723810017109\n",
      "fps: 8.664216734318677\n",
      "TIMESTEP 260086 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.749294 / Loss  0.019088171422481537\n",
      "fps: 7.512742435857938\n",
      "TIMESTEP 260087 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.378134 / Loss  0.008045964874327183\n",
      "fps: 7.511518079884238\n",
      "TIMESTEP 260088 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.27153 / Loss  0.014782199636101723\n",
      "fps: 8.730712227939584\n",
      "TIMESTEP 260089 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.823024 / Loss  0.023250378668308258\n",
      "fps: 6.966495478916003\n",
      "TIMESTEP 260090 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.38255 / Loss  0.02990998700261116\n",
      "fps: 9.027638412679964\n",
      "TIMESTEP 260091 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.20161 / Loss  0.02132239192724228\n",
      "fps: 8.502042247668392\n",
      "TIMESTEP 260092 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.752032 / Loss  0.02909553423523903\n",
      "fps: 7.5704265772383845\n",
      "TIMESTEP 260093 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.507492 / Loss  0.014759925194084644\n",
      "fps: 7.8003667431644805\n",
      "TIMESTEP 260094 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.367344 / Loss  0.013656163588166237\n",
      "fps: 8.841244345512877\n",
      "TIMESTEP 260095 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.350997 / Loss  0.016841743141412735\n",
      "fps: 8.204852131659358\n",
      "TIMESTEP 260096 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.341207 / Loss  0.017367033287882805\n",
      "fps: 8.693020180687308\n",
      "TIMESTEP 260097 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.04719 / Loss  0.09610000252723694\n",
      "fps: 8.52733061443564\n",
      "TIMESTEP 260098 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.350027 / Loss  0.04003516212105751\n",
      "fps: 8.213753613084654\n",
      "TIMESTEP 260099 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.483624 / Loss  0.010360239073634148\n",
      "fps: 8.497064531682394\n",
      "TIMESTEP 260100 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.448242 / Loss  0.029179608449339867\n",
      "fps: 8.879107665666764\n",
      "TIMESTEP 260101 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.975581 / Loss  0.028653249144554138\n",
      "fps: 7.661850803851473\n",
      "TIMESTEP 260102 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.375897 / Loss  0.2590532898902893\n",
      "fps: 5.401077561784673\n",
      "TIMESTEP 260103 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD -1 / Q_MAX  11.204436 / Loss  0.03210756555199623\n",
      "fps: 8.635941360566628\n",
      "TIMESTEP 260104 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.1462345 / Loss  0.015715181827545166\n",
      "fps: 8.665129616315872\n",
      "TIMESTEP 260105 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.04751 / Loss  0.02792978286743164\n",
      "fps: 8.740810206459061\n",
      "TIMESTEP 260106 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.386707 / Loss  0.019373269751667976\n",
      "fps: 8.649939986883732\n",
      "TIMESTEP 260107 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.247467 / Loss  0.009646839462220669\n",
      "fps: 8.57648446877294\n",
      "TIMESTEP 260108 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.222846 / Loss  0.04597104340791702\n",
      "fps: 8.700413416002705\n",
      "TIMESTEP 260109 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.268209 / Loss  0.048111025243997574\n",
      "fps: 8.601424443583134\n",
      "TIMESTEP 260110 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.018909 / Loss  0.007376215420663357\n",
      "fps: 8.423515046352083\n",
      "TIMESTEP 260111 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.992581 / Loss  0.014662024565041065\n",
      "fps: 8.661389810698127\n",
      "TIMESTEP 260112 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.249576 / Loss  0.08064232021570206\n",
      "fps: 8.716958388581071\n",
      "TIMESTEP 260113 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.227744 / Loss  0.007042553275823593\n",
      "fps: 8.544302276074632\n",
      "TIMESTEP 260114 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.186212 / Loss  0.033996954560279846\n",
      "fps: 8.921640322721307\n",
      "TIMESTEP 260115 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.090263 / Loss  0.013193199411034584\n",
      "fps: 8.666866413057628\n",
      "TIMESTEP 260116 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.243838 / Loss  0.024246226996183395\n",
      "fps: 8.610889501594151\n",
      "TIMESTEP 260117 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.292614 / Loss  0.013300718739628792\n",
      "fps: 8.998219365579263\n",
      "TIMESTEP 260118 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.223028 / Loss  0.043070532381534576\n",
      "fps: 8.742030822074842\n",
      "TIMESTEP 260119 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.060661 / Loss  0.01996440440416336\n",
      "fps: 8.563631500609455\n",
      "TIMESTEP 260120 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.140709 / Loss  0.010242423973977566\n",
      "fps: 9.176442275085162\n",
      "TIMESTEP 260121 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.254063 / Loss  0.46925824880599976\n",
      "fps: 8.899341190948537\n",
      "TIMESTEP 260122 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.930828 / Loss  0.019757449626922607\n",
      "fps: 8.792205396521943\n",
      "TIMESTEP 260123 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.909745 / Loss  0.02424856275320053\n",
      "fps: 8.754109583322897\n",
      "TIMESTEP 260124 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.297636 / Loss  0.01928265392780304\n",
      "fps: 8.789202260635232\n",
      "TIMESTEP 260125 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.759695 / Loss  0.015083380974829197\n",
      "fps: 8.617364559599118\n",
      "TIMESTEP 260126 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.225495 / Loss  0.024962972849607468\n",
      "fps: 9.084459422873245\n",
      "TIMESTEP 260127 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.388841 / Loss  0.01634632796049118\n",
      "fps: 8.702381051181396\n",
      "TIMESTEP 260128 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.442437 / Loss  0.44020503759384155\n",
      "fps: 8.95015886767571\n",
      "TIMESTEP 260129 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.170671 / Loss  0.01687445677816868\n",
      "fps: 8.866962915357718\n",
      "TIMESTEP 260130 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.060462 / Loss  0.023062502965331078\n",
      "fps: 8.717719340544226\n",
      "TIMESTEP 260131 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.226135 / Loss  0.010294603183865547\n",
      "fps: 8.581099217650332\n",
      "TIMESTEP 260132 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.009399 / Loss  0.01979159191250801\n",
      "fps: 9.072453235044732\n",
      "TIMESTEP 260133 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.890987 / Loss  0.010065126232802868\n",
      "fps: 8.81245167579924\n",
      "TIMESTEP 260134 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.063185 / Loss  0.04348619654774666\n",
      "fps: 8.716433046893671\n",
      "TIMESTEP 260135 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.253731 / Loss  0.016629917547106743\n",
      "fps: 9.001463651230369\n",
      "TIMESTEP 260136 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.796267 / Loss  0.06232558190822601\n",
      "fps: 8.989174766124798\n",
      "TIMESTEP 260137 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.484309 / Loss  0.04312105476856232\n",
      "fps: 8.811674096732311\n",
      "TIMESTEP 260138 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.572908 / Loss  0.02165452390909195\n",
      "fps: 6.191987906273621\n",
      "TIMESTEP 260139 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.803144 / Loss  0.027941733598709106\n",
      "fps: 8.61906454788125\n",
      "TIMESTEP 260140 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.832617 / Loss  0.01943664252758026\n",
      "fps: 8.32541807679711\n",
      "TIMESTEP 260141 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.027137 / Loss  0.040397223085165024\n",
      "fps: 7.43316396966695\n",
      "TIMESTEP 260142 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.020172 / Loss  0.027109021320939064\n",
      "fps: 7.348076306536581\n",
      "TIMESTEP 260143 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.234454 / Loss  0.010082770138978958\n",
      "fps: 7.566807323792222\n",
      "TIMESTEP 260144 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.083489 / Loss  0.026974007487297058\n",
      "fps: 7.7245159175374685\n",
      "TIMESTEP 260145 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.421583 / Loss  3.6127493381500244\n",
      "fps: 7.433664580641617\n",
      "TIMESTEP 260146 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.740525 / Loss  4.23028564453125\n",
      "fps: 8.076950777306838\n",
      "TIMESTEP 260147 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.26671 / Loss  0.09161461889743805\n",
      "fps: 9.154051476349382\n",
      "TIMESTEP 260148 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.012442 / Loss  0.02247615158557892\n",
      "fps: 8.790031142202086\n",
      "TIMESTEP 260149 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.219382 / Loss  0.02801966853439808\n",
      "fps: 8.729440265735372\n",
      "TIMESTEP 260150 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.7934 / Loss  0.042108431458473206\n",
      "fps: 8.846036550000527\n",
      "TIMESTEP 260151 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.529188 / Loss  0.03223242238163948\n",
      "fps: 8.867675289859362\n",
      "TIMESTEP 260152 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.653375 / Loss  0.010734936222434044\n",
      "fps: 8.736822809911846\n",
      "TIMESTEP 260153 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.81271 / Loss  0.10908517241477966\n",
      "fps: 8.015824085098414\n",
      "TIMESTEP 260154 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.680396 / Loss  0.01598830707371235\n",
      "fps: 8.801264067047594\n",
      "TIMESTEP 260155 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.377136 / Loss  0.012864555232226849\n",
      "fps: 8.737132203601671\n",
      "TIMESTEP 260156 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  9.614819 / Loss  0.0614926852285862\n",
      "fps: 7.733959282567769\n",
      "TIMESTEP 260157 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.625065 / Loss  0.03822369500994682\n",
      "fps: 8.799159168483083\n",
      "TIMESTEP 260158 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.432605 / Loss  0.04631566256284714\n",
      "fps: 8.46891020657878\n",
      "TIMESTEP 260159 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.771302 / Loss  0.03945165500044823\n",
      "fps: 9.15648952450395\n",
      "TIMESTEP 260160 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.312177 / Loss  0.03564884886145592\n",
      "fps: 8.84674556587437\n",
      "TIMESTEP 260161 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.355097 / Loss  0.4808555245399475\n",
      "fps: 7.606769062244396\n",
      "TIMESTEP 260162 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.391947 / Loss  0.05552899092435837\n",
      "fps: 7.638896173907976\n",
      "TIMESTEP 260163 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.207925 / Loss  0.012940808199346066\n",
      "fps: 7.639717092309289\n",
      "TIMESTEP 260164 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.149354 / Loss  0.012552336789667606\n",
      "fps: 7.172509084690693\n",
      "TIMESTEP 260165 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.102465 / Loss  0.054823845624923706\n",
      "fps: 9.054591299241835\n",
      "TIMESTEP 260166 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.486873 / Loss  0.021077526733279228\n",
      "fps: 7.554582132564842\n",
      "TIMESTEP 260167 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.770925 / Loss  0.025547489523887634\n",
      "fps: 8.094549216078308\n",
      "TIMESTEP 260168 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.738318 / Loss  0.059317074716091156\n",
      "fps: 7.609018409805017\n",
      "TIMESTEP 260169 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.209102 / Loss  0.06440482288599014\n",
      "fps: 7.74372598723504\n",
      "TIMESTEP 260170 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.524886 / Loss  0.04572825878858566\n",
      "fps: 6.854479520580742\n",
      "TIMESTEP 260171 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.687991 / Loss  0.0740836039185524\n",
      "fps: 9.07661545120104\n",
      "TIMESTEP 260172 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.827491 / Loss  0.060045890510082245\n",
      "fps: 7.630502332283018\n",
      "TIMESTEP 260173 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.648458 / Loss  0.5985593199729919\n",
      "fps: 7.030321722019035\n",
      "TIMESTEP 260174 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.073595 / Loss  0.039168134331703186\n",
      "fps: 5.2741225836549255\n",
      "TIMESTEP 260175 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.131863 / Loss  0.03893878683447838\n",
      "fps: 8.433694530962526\n",
      "TIMESTEP 260176 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.429216 / Loss  0.04054615646600723\n",
      "fps: 8.684632379280142\n",
      "TIMESTEP 260177 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.268898 / Loss  0.03677326440811157\n",
      "fps: 9.023462744878685\n",
      "TIMESTEP 260178 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.101252 / Loss  0.023526541888713837\n",
      "fps: 8.818250816270323\n",
      "TIMESTEP 260179 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.732561 / Loss  0.046194158494472504\n",
      "fps: 8.534739815114389\n",
      "TIMESTEP 260180 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.646929 / Loss  0.046327266842126846\n",
      "fps: 8.774015400481552\n",
      "TIMESTEP 260181 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.387206 / Loss  0.05438225716352463\n",
      "fps: 8.598074285749428\n",
      "TIMESTEP 260182 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.300121 / Loss  0.02993481419980526\n",
      "fps: 8.771299810742704\n",
      "TIMESTEP 260183 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.515064 / Loss  0.04659992828965187\n",
      "fps: 8.014169951524856\n",
      "TIMESTEP 260184 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.3929 / Loss  0.02540443278849125\n",
      "fps: 8.853075767042096\n",
      "TIMESTEP 260185 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.6338415 / Loss  0.05395763739943504\n",
      "fps: 8.644306588899651\n",
      "TIMESTEP 260186 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.087766 / Loss  0.15579411387443542\n",
      "fps: 9.199871026893526\n",
      "TIMESTEP 260187 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.760912 / Loss  0.14920493960380554\n",
      "fps: 8.833461101048819\n",
      "TIMESTEP 260188 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.456961 / Loss  0.27905866503715515\n",
      "fps: 8.70745768552869\n",
      "TIMESTEP 260189 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.8816595 / Loss  0.17713521420955658\n",
      "fps: 8.956714486446318\n",
      "TIMESTEP 260190 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.735846 / Loss  0.255471408367157\n",
      "fps: 9.021192027341222\n",
      "TIMESTEP 260191 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  13.294134 / Loss  0.885103166103363\n",
      "fps: 7.135366498871251\n",
      "TIMESTEP 260192 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.937307 / Loss  0.10164342820644379\n",
      "fps: 9.06658942394366\n",
      "TIMESTEP 260193 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  13.236509 / Loss  0.12735940515995026\n",
      "fps: 8.609705230314475\n",
      "TIMESTEP 260194 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.826294 / Loss  2.7198283672332764\n",
      "fps: 8.918472274789812\n",
      "TIMESTEP 260195 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.763235 / Loss  0.03946436941623688\n",
      "fps: 9.077519077856701\n",
      "TIMESTEP 260196 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.899565 / Loss  0.5622940063476562\n",
      "fps: 9.054200378202417\n",
      "TIMESTEP 260197 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  13.405318 / Loss  0.09408502280712128\n",
      "fps: 8.8124331604171\n",
      "TIMESTEP 260198 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.944088 / Loss  0.021756542846560478\n",
      "fps: 9.059774191667081\n",
      "TIMESTEP 260199 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.888426 / Loss  0.05699503794312477\n",
      "fps: 8.816397226210169\n",
      "TIMESTEP 260200 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.647784 / Loss  0.062387410551309586\n",
      "fps: 8.764096970608762\n",
      "TIMESTEP 260201 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.614175 / Loss  0.050863638520240784\n",
      "fps: 8.737441619205146\n",
      "TIMESTEP 260202 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.568508 / Loss  0.40691205859184265\n",
      "fps: 7.838692375116104\n",
      "TIMESTEP 260203 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.536782 / Loss  0.050230931490659714\n",
      "fps: 7.6203820452538675\n",
      "TIMESTEP 260204 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  13.033376 / Loss  0.026460446417331696\n",
      "fps: 9.293147188439578\n",
      "TIMESTEP 260205 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.592748 / Loss  0.07611514627933502\n",
      "fps: 7.754520831600063\n",
      "TIMESTEP 260206 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.689779 / Loss  0.11790096759796143\n",
      "fps: 7.908140986240002\n",
      "TIMESTEP 260207 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.627293 / Loss  0.05540970340371132\n",
      "fps: 7.711421784566507\n",
      "TIMESTEP 260208 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.357612 / Loss  0.02106073684990406\n",
      "fps: 7.653155101094975\n",
      "TIMESTEP 260209 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  1.8340166 / Loss  0.016630586236715317\n",
      "fps: 7.0069262304667275\n",
      "TIMESTEP 260210 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.4675865 / Loss  0.07650645822286606\n",
      "fps: 6.1448791257733655\n",
      "TIMESTEP 260211 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.797744 / Loss  0.02518630400300026\n",
      "fps: 8.971984367546616\n",
      "TIMESTEP 260212 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.681562 / Loss  0.08276847749948502\n",
      "fps: 8.721580500342059\n",
      "TIMESTEP 260213 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.396459 / Loss  0.027761995792388916\n",
      "fps: 9.119281294435337\n",
      "TIMESTEP 260214 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  13.163151 / Loss  0.03916584327816963\n",
      "fps: 8.746223590154226\n",
      "TIMESTEP 260215 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.754097 / Loss  2.208235740661621\n",
      "fps: 8.495962988320416\n",
      "TIMESTEP 260216 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  13.040781 / Loss  0.035423025488853455\n",
      "fps: 7.787795177635757\n",
      "TIMESTEP 260217 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.971547 / Loss  0.021130673587322235\n",
      "fps: 8.795616351027444\n",
      "TIMESTEP 260218 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.886158 / Loss  0.06828934699296951\n",
      "fps: 8.417429277847349\n",
      "TIMESTEP 260219 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.1538315 / Loss  0.045963309705257416\n",
      "fps: 8.837276529397512\n",
      "TIMESTEP 260220 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.66953 / Loss  0.0755937248468399\n",
      "fps: 8.839306563036084\n",
      "TIMESTEP 260221 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.724928 / Loss  0.055037785321474075\n",
      "fps: 8.651938397052696\n",
      "TIMESTEP 260222 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.627106 / Loss  0.07026762515306473\n",
      "fps: 8.971427838072545\n",
      "TIMESTEP 260223 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.636902 / Loss  0.2087140679359436\n",
      "fps: 8.353107294000498\n",
      "TIMESTEP 260224 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.834471 / Loss  0.08820800483226776\n",
      "fps: 8.325715544774772\n",
      "TIMESTEP 260225 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  13.099321 / Loss  3.303806781768799\n",
      "fps: 7.595212139869258\n",
      "TIMESTEP 260226 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.851863 / Loss  0.04234326258301735\n",
      "fps: 8.724864268924344\n",
      "TIMESTEP 260227 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.317142 / Loss  0.37469482421875\n",
      "fps: 8.691272887017497\n",
      "TIMESTEP 260228 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.492998 / Loss  0.039141684770584106\n",
      "fps: 8.631835871178055\n",
      "TIMESTEP 260229 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.285316 / Loss  0.02668885327875614\n",
      "fps: 8.656205950761851\n",
      "TIMESTEP 260230 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.055995 / Loss  0.09674072265625\n",
      "fps: 8.551008758338362\n",
      "TIMESTEP 260231 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.269234 / Loss  0.053462591022253036\n",
      "fps: 9.061927324030083\n",
      "TIMESTEP 260232 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.719169 / Loss  0.03427303209900856\n",
      "fps: 8.630059813461012\n",
      "TIMESTEP 260233 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.75381 / Loss  0.05756895989179611\n",
      "fps: 8.54247506079504\n",
      "TIMESTEP 260234 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.940686 / Loss  0.04962558671832085\n",
      "fps: 8.840461677247474\n",
      "TIMESTEP 260235 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.822634 / Loss  0.037597302347421646\n",
      "fps: 8.928876456638267\n",
      "TIMESTEP 260236 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.603858 / Loss  0.027533158659934998\n",
      "fps: 8.639890577557688\n",
      "TIMESTEP 260237 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.852283 / Loss  0.030755246058106422\n",
      "fps: 8.834075069452197\n",
      "TIMESTEP 260238 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.796922 / Loss  0.022286275401711464\n",
      "fps: 8.540666017780564\n",
      "TIMESTEP 260239 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.9707 / Loss  0.03839808702468872\n",
      "fps: 8.53255215485236\n",
      "TIMESTEP 260240 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.702047 / Loss  0.013769900426268578\n",
      "fps: 8.510270831811578\n",
      "TIMESTEP 260241 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.637395 / Loss  0.05345246195793152\n",
      "fps: 7.813648489545299\n",
      "TIMESTEP 260242 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.21878 / Loss  0.02207339182496071\n",
      "fps: 6.906125325399165\n",
      "TIMESTEP 260243 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.174195 / Loss  0.04121626541018486\n",
      "fps: 7.931094861802085\n",
      "TIMESTEP 260244 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.477512 / Loss  0.030994411557912827\n",
      "fps: 7.53643333686709\n",
      "TIMESTEP 260245 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.4293585 / Loss  0.033580876886844635\n",
      "fps: 8.135701580663495\n",
      "TIMESTEP 260246 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.312809 / Loss  0.1757655143737793\n",
      "fps: 6.030596517058133\n",
      "TIMESTEP 260247 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.633998 / Loss  0.054286982864141464\n",
      "fps: 8.946302612241938\n",
      "TIMESTEP 260248 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.477487 / Loss  0.022571071982383728\n",
      "fps: 7.529384731930966\n",
      "TIMESTEP 260249 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.368334 / Loss  0.08431252837181091\n",
      "fps: 7.959752457115529\n",
      "TIMESTEP 260250 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.133892 / Loss  0.4824537932872772\n",
      "fps: 7.434784612493929\n",
      "TIMESTEP 260251 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.886746 / Loss  0.05272693186998367\n",
      "fps: 8.036774103305492\n",
      "TIMESTEP 260252 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.407554 / Loss  0.027277911081910133\n",
      "fps: 8.106219729503032\n",
      "TIMESTEP 260253 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.748211 / Loss  0.01959931291639805\n",
      "fps: 8.281920302661119\n",
      "TIMESTEP 260254 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.958709 / Loss  0.021705294027924538\n",
      "fps: 8.577414037800056\n",
      "TIMESTEP 260255 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.6806 / Loss  0.05803796276450157\n",
      "fps: 9.118190928992398\n",
      "TIMESTEP 260256 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.887782 / Loss  0.04237116873264313\n",
      "fps: 8.82714033167775\n",
      "TIMESTEP 260257 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.770394 / Loss  0.12608647346496582\n",
      "fps: 8.701406148216071\n",
      "TIMESTEP 260258 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.665486 / Loss  0.019228022545576096\n",
      "fps: 7.8050552399312965\n",
      "TIMESTEP 260259 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.360815 / Loss  0.026268552988767624\n",
      "fps: 8.865463553622233\n",
      "TIMESTEP 260260 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.294202 / Loss  0.025334378704428673\n",
      "fps: 8.624540942009615\n",
      "TIMESTEP 260261 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.169627 / Loss  0.03778048977255821\n",
      "fps: 9.204635780482935\n",
      "TIMESTEP 260262 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.555274 / Loss  0.020702185109257698\n",
      "fps: 8.80213216931439\n",
      "TIMESTEP 260263 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.356438 / Loss  0.015110206790268421\n",
      "fps: 8.74543941918144\n",
      "TIMESTEP 260264 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.405147 / Loss  0.02927841618657112\n",
      "fps: 7.792598868167107\n",
      "TIMESTEP 260265 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.144125 / Loss  0.0750921443104744\n",
      "fps: 8.895698391085434\n",
      "TIMESTEP 260266 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.34664 / Loss  2.9771885871887207\n",
      "fps: 7.4064792283975684\n",
      "TIMESTEP 260267 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.317819 / Loss  0.012212244793772697\n",
      "fps: 9.053535889897642\n",
      "TIMESTEP 260268 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.04961 / Loss  0.019212692975997925\n",
      "fps: 8.51848075763084\n",
      "TIMESTEP 260269 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.855808 / Loss  0.013098908588290215\n",
      "fps: 8.477468828257798\n",
      "TIMESTEP 260270 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.553077 / Loss  0.026833590120077133\n",
      "fps: 7.6442701289991\n",
      "TIMESTEP 260271 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.937511 / Loss  0.014134892262518406\n",
      "fps: 8.895528592364482\n",
      "TIMESTEP 260272 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.651276 / Loss  0.04696786031126976\n",
      "fps: 7.961157529904375\n",
      "TIMESTEP 260273 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.744939 / Loss  0.01628369465470314\n",
      "fps: 8.81095218188655\n",
      "TIMESTEP 260274 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.942276 / Loss  0.3158706724643707\n",
      "fps: 8.623211080660449\n",
      "TIMESTEP 260275 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.520018 / Loss  0.022125739604234695\n",
      "fps: 7.795379611560264\n",
      "TIMESTEP 260276 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.65793 / Loss  0.06503885239362717\n",
      "fps: 8.737441619205146\n",
      "TIMESTEP 260277 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.512648 / Loss  0.02475684881210327\n",
      "fps: 8.93321235820565\n",
      "TIMESTEP 260278 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.690183 / Loss  3.1958892345428467\n",
      "fps: 7.87977914170363\n",
      "TIMESTEP 260279 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.424203 / Loss  0.03376588225364685\n",
      "fps: 9.105146618271494\n",
      "TIMESTEP 260280 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.383336 / Loss  0.023206591606140137\n",
      "fps: 7.647545541898912\n",
      "TIMESTEP 260281 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.223206 / Loss  0.046299710869789124\n",
      "fps: 7.958680100946851\n",
      "TIMESTEP 260282 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.002978 / Loss  0.018532579764723778\n",
      "fps: 5.20548509643846\n",
      "TIMESTEP 260283 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.078741 / Loss  0.0625089779496193\n",
      "fps: 8.819660066069059\n",
      "TIMESTEP 260284 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.730772 / Loss  0.04796802997589111\n",
      "fps: 8.573977391197694\n",
      "TIMESTEP 260285 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.255866 / Loss  0.019645828753709793\n",
      "fps: 9.100425046269075\n",
      "TIMESTEP 260286 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.251906 / Loss  0.031921062618494034\n",
      "fps: 8.87003819296116\n",
      "TIMESTEP 260287 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  9.677777 / Loss  1.7350585460662842\n",
      "fps: 8.8932275156956\n",
      "TIMESTEP 260288 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.061725 / Loss  0.10118377208709717\n",
      "fps: 7.794727696782167\n",
      "TIMESTEP 260289 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.263657 / Loss  0.03167836368083954\n",
      "fps: 9.019329727869945\n",
      "TIMESTEP 260290 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.355099 / Loss  0.04444717615842819\n",
      "fps: 8.590853882092215\n",
      "TIMESTEP 260291 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.353058 / Loss  0.026096787303686142\n",
      "fps: 9.130318558411084\n",
      "TIMESTEP 260292 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.24767 / Loss  0.028131552040576935\n",
      "fps: 8.939038217266395\n",
      "TIMESTEP 260293 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  9.924223 / Loss  0.03593560308218002\n",
      "fps: 8.956637980738432\n",
      "TIMESTEP 260294 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.701983 / Loss  0.03624185919761658\n",
      "fps: 8.96282002790795\n",
      "TIMESTEP 260295 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.54466 / Loss  0.01800413429737091\n",
      "fps: 8.831898657412866\n",
      "TIMESTEP 260296 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.364744 / Loss  0.020969586446881294\n",
      "fps: 8.62642117906445\n",
      "TIMESTEP 260297 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.324608 / Loss  0.01436060294508934\n",
      "fps: 9.063043440681774\n",
      "TIMESTEP 260298 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.141174 / Loss  0.01721661165356636\n",
      "fps: 8.663321938675397\n",
      "TIMESTEP 260299 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.477605 / Loss  0.05118212848901749\n",
      "fps: 8.68968832094407\n",
      "TIMESTEP 260300 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.649092 / Loss  0.02415144257247448\n",
      "fps: 7.520447536398192\n",
      "TIMESTEP 260301 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.623023 / Loss  0.026135284453630447\n",
      "fps: 8.843835593331127\n",
      "TIMESTEP 260302 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.67849 / Loss  1.5849778652191162\n",
      "fps: 8.599502191750489\n",
      "TIMESTEP 260303 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.391702 / Loss  0.025371812283992767\n",
      "fps: 7.672474568706109\n",
      "TIMESTEP 260304 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.324965 / Loss  0.0190291590988636\n",
      "fps: 8.628728249733586\n",
      "TIMESTEP 260305 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.595033 / Loss  0.05031987279653549\n",
      "fps: 8.66162233603172\n",
      "TIMESTEP 260306 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.527554 / Loss  0.07292507588863373\n",
      "fps: 8.546670117146508\n",
      "TIMESTEP 260307 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.5802 / Loss  0.0120901670306921\n",
      "fps: 7.706476903374693\n",
      "TIMESTEP 260308 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.673162 / Loss  0.021667536348104477\n",
      "fps: 8.426171874215251\n",
      "TIMESTEP 260309 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.5668335 / Loss  0.021924477070569992\n",
      "fps: 9.11444602352982\n",
      "TIMESTEP 260310 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.868895 / Loss  0.006685530301183462\n",
      "fps: 7.023799476183778\n",
      "TIMESTEP 260311 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.537217 / Loss  0.021432651206851006\n",
      "fps: 8.621509456456723\n",
      "TIMESTEP 260312 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.74448 / Loss  0.021391279995441437\n",
      "fps: 8.325137154335348\n",
      "TIMESTEP 260313 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.765462 / Loss  0.06953511387109756\n",
      "fps: 8.59904379987043\n",
      "TIMESTEP 260314 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.503407 / Loss  0.5597771406173706\n",
      "fps: 4.120870094908726\n",
      "TIMESTEP 260315 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.581656 / Loss  0.010686454363167286\n",
      "fps: 5.759136164726269\n",
      "TIMESTEP 260316 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.770997 / Loss  0.027972809970378876\n",
      "fps: 7.984101482491072\n",
      "TIMESTEP 260317 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.732167 / Loss  0.02609109878540039\n",
      "fps: 6.356306214149864\n",
      "TIMESTEP 260318 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.567477 / Loss  0.02820727415382862\n",
      "fps: 5.631616401128927\n",
      "TIMESTEP 260319 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.766599 / Loss  0.02546628750860691\n",
      "fps: 8.763309591513954\n",
      "TIMESTEP 260320 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.955976 / Loss  0.011875982396304607\n",
      "fps: 7.991357613055034\n",
      "TIMESTEP 260321 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.931701 / Loss  2.3599884510040283\n",
      "fps: 7.6052656391659115\n",
      "TIMESTEP 260322 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.785472 / Loss  0.03702888637781143\n",
      "fps: 7.445844901900914\n",
      "TIMESTEP 260323 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.897628 / Loss  0.02235465869307518\n",
      "fps: 7.220578361870397\n",
      "TIMESTEP 260324 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.96785 / Loss  0.04172850027680397\n",
      "fps: 7.401382055861032\n",
      "TIMESTEP 260325 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.076203 / Loss  0.028351787477731705\n",
      "fps: 7.466136919097791\n",
      "TIMESTEP 260326 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.159004 / Loss  0.01747594028711319\n",
      "fps: 6.966472337103659\n",
      "TIMESTEP 260327 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.938281 / Loss  0.03997323662042618\n",
      "fps: 8.492780444937605\n",
      "TIMESTEP 260328 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.034554 / Loss  0.03437734395265579\n",
      "fps: 8.028712914877692\n",
      "TIMESTEP 260329 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.89436 / Loss  0.02462039887905121\n",
      "fps: 8.04906619176652\n",
      "TIMESTEP 260330 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.2197075 / Loss  0.025707773864269257\n",
      "fps: 8.13199477294495\n",
      "TIMESTEP 260331 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.044775 / Loss  0.13629849255084991\n",
      "fps: 7.277482041850297\n",
      "TIMESTEP 260332 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.031301 / Loss  0.029180167242884636\n",
      "fps: 8.0147671987801\n",
      "TIMESTEP 260333 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.559838 / Loss  0.020549975335597992\n",
      "fps: 7.490644516634847\n",
      "TIMESTEP 260334 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.166539 / Loss  0.028609059751033783\n",
      "fps: 6.867318419747103\n",
      "TIMESTEP 260335 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.464823 / Loss  0.04332245886325836\n",
      "fps: 8.256910281017767\n",
      "TIMESTEP 260336 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.48234 / Loss  0.030665528029203415\n",
      "fps: 8.193952465235856\n",
      "TIMESTEP 260337 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.690315 / Loss  0.033839110285043716\n",
      "fps: 8.399830173111276\n",
      "TIMESTEP 260338 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.421436 / Loss  0.05572879686951637\n",
      "fps: 8.051785700573605\n",
      "TIMESTEP 260339 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.551326 / Loss  0.02685457095503807\n",
      "fps: 8.498493521229497\n",
      "TIMESTEP 260340 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.591005 / Loss  0.036186568439006805\n",
      "fps: 7.139848497744489\n",
      "TIMESTEP 260341 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.669347 / Loss  0.03130519390106201\n",
      "fps: 8.022017829239417\n",
      "TIMESTEP 260342 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.628706 / Loss  0.10082659125328064\n",
      "fps: 8.208641738524587\n",
      "TIMESTEP 260343 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.64446 / Loss  0.03372706100344658\n",
      "fps: 7.220901565964714\n",
      "TIMESTEP 260344 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.882334 / Loss  0.02870606817305088\n",
      "fps: 7.853869444969787\n",
      "TIMESTEP 260345 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.994075 / Loss  0.03524202108383179\n",
      "fps: 8.568092399586538\n",
      "TIMESTEP 260346 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.799539 / Loss  0.013346921652555466\n",
      "fps: 8.140154213107532\n",
      "TIMESTEP 260347 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.512281 / Loss  0.03146488964557648\n",
      "fps: 6.949942170862731\n",
      "TIMESTEP 260348 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.05178 / Loss  0.03346533700823784\n",
      "fps: 8.022462797903676\n",
      "TIMESTEP 260349 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.300654 / Loss  0.027907542884349823\n",
      "fps: 8.530244174269573\n",
      "TIMESTEP 260350 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.879283 / Loss  0.026887308806180954\n",
      "fps: 7.7166995378411\n",
      "TIMESTEP 260351 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.905605 / Loss  0.06252408027648926\n",
      "fps: 8.056719497038976\n",
      "TIMESTEP 260352 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.2620125 / Loss  0.028332386165857315\n",
      "fps: 8.216939695873805\n",
      "TIMESTEP 260353 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.869529 / Loss  0.050468601286411285\n",
      "fps: 8.19800439775226\n",
      "TIMESTEP 260354 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.096213 / Loss  0.022687947377562523\n",
      "fps: 5.607013472433514\n",
      "TIMESTEP 260355 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.056061 / Loss  0.02731684222817421\n",
      "fps: 8.37163033368329\n",
      "TIMESTEP 260356 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.348621 / Loss  0.0365656316280365\n",
      "fps: 8.060296022414992\n",
      "TIMESTEP 260357 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.044183 / Loss  0.15250945091247559\n",
      "fps: 8.510426241216777\n",
      "TIMESTEP 260358 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.033827 / Loss  0.11266429722309113\n",
      "fps: 8.183911993225445\n",
      "TIMESTEP 260359 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.509102 / Loss  0.021488524973392487\n",
      "fps: 8.136822174628302\n",
      "TIMESTEP 260360 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.368535 / Loss  0.015195338986814022\n",
      "fps: 8.211116505614633\n",
      "TIMESTEP 260361 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.408737 / Loss  0.02046470157802105\n",
      "fps: 7.278012417186941\n",
      "TIMESTEP 260362 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.40979 / Loss  0.04413548856973648\n",
      "fps: 7.885571615503783\n",
      "TIMESTEP 260363 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.540009 / Loss  0.04590708017349243\n",
      "fps: 8.566272491473153\n",
      "TIMESTEP 260364 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.360698 / Loss  0.07130683958530426\n",
      "fps: 8.252881117147394\n",
      "TIMESTEP 260365 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.647639 / Loss  0.022473283112049103\n",
      "fps: 8.267130780314066\n",
      "TIMESTEP 260366 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.694322 / Loss  0.021365664899349213\n",
      "fps: 8.159314931786923\n",
      "TIMESTEP 260367 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.5218525 / Loss  0.025499772280454636\n",
      "fps: 8.3817349773186\n",
      "TIMESTEP 260368 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.289416 / Loss  0.04073233902454376\n",
      "fps: 7.957985489207978\n",
      "TIMESTEP 260369 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.145158 / Loss  0.017441600561141968\n",
      "fps: 8.519709286500673\n",
      "TIMESTEP 260370 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.092221 / Loss  0.12154503911733627\n",
      "fps: 7.090038997459334\n",
      "TIMESTEP 260371 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.622083 / Loss  0.03795544058084488\n",
      "fps: 8.247833969803395\n",
      "TIMESTEP 260372 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.07862 / Loss  0.023076001554727554\n",
      "fps: 8.209846795866797\n",
      "TIMESTEP 260373 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.861664 / Loss  0.03374948352575302\n",
      "fps: 8.412566990789768\n",
      "TIMESTEP 260374 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.063268 / Loss  0.023702220991253853\n",
      "fps: 8.155475575207129\n",
      "TIMESTEP 260375 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.974959 / Loss  0.051737502217292786\n",
      "fps: 8.910287513330472\n",
      "TIMESTEP 260376 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.946853 / Loss  0.04527691379189491\n",
      "fps: 8.212740377555036\n",
      "TIMESTEP 260377 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.923198 / Loss  0.01704082265496254\n",
      "fps: 8.45528638845939\n",
      "TIMESTEP 260378 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.139866 / Loss  0.020357025787234306\n",
      "fps: 8.516560809603483\n",
      "TIMESTEP 260379 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.994776 / Loss  0.012990426272153854\n",
      "fps: 7.534497377308328\n",
      "TIMESTEP 260380 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.733956 / Loss  0.0355999618768692\n",
      "fps: 8.11115773840028\n",
      "TIMESTEP 260381 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.920933 / Loss  0.01874249055981636\n",
      "fps: 8.825060176067598\n",
      "TIMESTEP 260382 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.551522 / Loss  0.03089694306254387\n",
      "fps: 7.547567462516825\n",
      "TIMESTEP 260383 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.719832 / Loss  0.016798466444015503\n",
      "fps: 7.451056466884817\n",
      "TIMESTEP 260384 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  6.2493215 / Loss  0.6852498054504395\n",
      "fps: 8.063736196130302\n",
      "TIMESTEP 260385 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.521734 / Loss  0.009504100307822227\n",
      "fps: 8.660638660320672\n",
      "TIMESTEP 260386 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.363532 / Loss  5.596685409545898\n",
      "fps: 7.8677621459388485\n",
      "TIMESTEP 260387 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.037789 / Loss  0.04155212640762329\n",
      "fps: 8.808787146907488\n",
      "TIMESTEP 260388 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.130621 / Loss  0.038731276988983154\n",
      "fps: 8.461461180766399\n",
      "TIMESTEP 260389 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.642175 / Loss  0.027587121352553368\n",
      "fps: 6.244766999528029\n",
      "TIMESTEP 260390 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.885097 / Loss  0.3862375319004059\n",
      "fps: 5.1029196707310245\n",
      "TIMESTEP 260391 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.518767 / Loss  0.03510110452771187\n",
      "fps: 8.368206313008884\n",
      "TIMESTEP 260392 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.500566 / Loss  0.029155150055885315\n",
      "fps: 8.7446918522225\n",
      "TIMESTEP 260393 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.878391 / Loss  0.03790523111820221\n",
      "fps: 7.553724991580566\n",
      "TIMESTEP 260394 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.603944 / Loss  0.018471477553248405\n",
      "fps: 8.822851084686079\n",
      "TIMESTEP 260395 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.203162 / Loss  0.03523879870772362\n",
      "fps: 8.344748004488409\n",
      "TIMESTEP 260396 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.548093 / Loss  0.023779146373271942\n",
      "fps: 9.04414577399037\n",
      "TIMESTEP 260397 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  0.8808908 / Loss  0.020630622282624245\n",
      "fps: 7.48476299076161\n",
      "TIMESTEP 260398 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.319277 / Loss  0.02060227282345295\n",
      "fps: 8.5658176352378\n",
      "TIMESTEP 260399 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.27901 / Loss  0.036955613642930984\n",
      "fps: 7.364513483949017\n",
      "TIMESTEP 260400 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.985487 / Loss  0.054629649966955185\n",
      "fps: 7.611904489681844\n",
      "TIMESTEP 260401 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  9.758176 / Loss  0.05121903866529465\n",
      "fps: 8.15382671360836\n",
      "TIMESTEP 260402 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.50719 / Loss  0.017992503941059113\n",
      "fps: 8.933764156348712\n",
      "TIMESTEP 260403 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  9.915331 / Loss  0.025665732100605965\n",
      "fps: 8.658082148799227\n",
      "TIMESTEP 260404 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.399156 / Loss  0.09616214781999588\n",
      "fps: 8.644092807003805\n",
      "TIMESTEP 260405 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.445389 / Loss  0.03412635624408722\n",
      "fps: 8.538979427722188\n",
      "TIMESTEP 260406 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.53961 / Loss  0.029975876212120056\n",
      "fps: 7.612553813986918\n",
      "TIMESTEP 260407 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.662953 / Loss  0.02947080135345459\n",
      "fps: 8.419000592137618\n",
      "TIMESTEP 260408 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.249188 / Loss  0.027573423460125923\n",
      "fps: 8.991679975989625\n",
      "TIMESTEP 260409 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.289976 / Loss  0.02362552471458912\n",
      "fps: 8.684704308700846\n",
      "TIMESTEP 260410 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.573744 / Loss  0.03904926776885986\n",
      "fps: 8.625002056360735\n",
      "TIMESTEP 260411 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.81933 / Loss  0.010822147130966187\n",
      "fps: 8.699348736881404\n",
      "TIMESTEP 260412 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.634815 / Loss  0.025760045275092125\n",
      "fps: 8.854178409107702\n",
      "TIMESTEP 260413 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.618671 / Loss  0.016920315101742744\n",
      "fps: 8.191760007031045\n",
      "TIMESTEP 260414 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.705613 / Loss  0.0165582112967968\n",
      "fps: 7.745813888637526\n",
      "TIMESTEP 260415 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.599788 / Loss  0.028540780767798424\n",
      "fps: 8.412128310756875\n",
      "TIMESTEP 260416 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.727473 / Loss  0.03419007360935211\n",
      "fps: 8.628497487137393\n",
      "TIMESTEP 260417 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.586358 / Loss  0.13208261132240295\n",
      "fps: 7.665939850311166\n",
      "TIMESTEP 260418 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.64168 / Loss  0.010891702957451344\n",
      "fps: 8.837444111536728\n",
      "TIMESTEP 260419 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.591882 / Loss  0.10790731012821198\n",
      "fps: 8.414474093362998\n",
      "TIMESTEP 260420 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.842006 / Loss  0.0136744175106287\n",
      "fps: 8.941725239889058\n",
      "TIMESTEP 260421 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.749415 / Loss  0.01000042725354433\n",
      "fps: 8.689004255107578\n",
      "TIMESTEP 260422 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.997445 / Loss  0.04334254562854767\n",
      "fps: 7.640524268838065\n",
      "TIMESTEP 260423 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.751333 / Loss  0.025172241032123566\n",
      "fps: 8.61393121266078\n",
      "TIMESTEP 260424 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.842377 / Loss  0.020548326894640923\n",
      "fps: 7.981184493952703\n",
      "TIMESTEP 260425 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.740561 / Loss  0.020569197833538055\n",
      "fps: 7.206683138545916\n",
      "TIMESTEP 260426 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.022958 / Loss  0.1749800592660904\n",
      "fps: 5.440555794083273\n",
      "TIMESTEP 260427 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.883218 / Loss  0.006982040591537952\n",
      "fps: 8.904328922535406\n",
      "TIMESTEP 260428 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.866716 / Loss  0.01312870904803276\n",
      "fps: 8.644538197886632\n",
      "TIMESTEP 260429 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.719382 / Loss  0.016056567430496216\n",
      "fps: 8.799731453507889\n",
      "TIMESTEP 260430 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.145227 / Loss  0.1092720553278923\n",
      "fps: 8.63206681237626\n",
      "TIMESTEP 260431 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.937972 / Loss  0.020816657692193985\n",
      "fps: 8.663787209343157\n",
      "TIMESTEP 260432 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.047931 / Loss  0.019421573728322983\n",
      "fps: 8.9394383097715\n",
      "TIMESTEP 260433 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.13631 / Loss  0.00582543108612299\n",
      "fps: 8.545033381073939\n",
      "TIMESTEP 260434 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.977089 / Loss  0.018837371841073036\n",
      "fps: 8.480022644103435\n",
      "TIMESTEP 260435 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.894364 / Loss  0.016422897577285767\n",
      "fps: 7.849357162908206\n",
      "TIMESTEP 260436 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.125703 / Loss  0.0197763592004776\n",
      "fps: 8.602165357156775\n",
      "TIMESTEP 260437 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.206894 / Loss  0.024157052859663963\n",
      "fps: 7.607458904743162\n",
      "TIMESTEP 260438 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.883785 / Loss  0.023588553071022034\n",
      "fps: 7.792729171195375\n",
      "TIMESTEP 260439 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.183935 / Loss  0.015512608923017979\n",
      "fps: 8.929941024931338\n",
      "TIMESTEP 260440 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.204303 / Loss  0.012269530445337296\n",
      "fps: 8.682277349054417\n",
      "TIMESTEP 260441 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.053522 / Loss  0.009071425534784794\n",
      "fps: 8.817861497708448\n",
      "TIMESTEP 260442 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.135428 / Loss  0.016245782375335693\n",
      "fps: 8.636563745122466\n",
      "TIMESTEP 260443 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.234313 / Loss  2.6215453147888184\n",
      "fps: 8.563474141979215\n",
      "TIMESTEP 260444 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.218336 / Loss  0.012670530937612057\n",
      "fps: 7.9523575685068515\n",
      "TIMESTEP 260445 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.356137 / Loss  0.02430197410285473\n",
      "fps: 8.087681545070998\n",
      "TIMESTEP 260446 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.49289 / Loss  0.03475248068571091\n",
      "fps: 7.588863236667077\n",
      "TIMESTEP 260447 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.010786 / Loss  0.05203863978385925\n",
      "fps: 8.13172675190725\n",
      "TIMESTEP 260448 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.446695 / Loss  0.015125645324587822\n",
      "fps: 7.53563446377624\n",
      "TIMESTEP 260449 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.95322 / Loss  0.03833587095141411\n",
      "fps: 8.808713147427094\n",
      "TIMESTEP 260450 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.898831 / Loss  0.03312365338206291\n",
      "fps: 8.96862704847925\n",
      "TIMESTEP 260451 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.9214325 / Loss  0.01790674775838852\n",
      "fps: 7.697820944116843\n",
      "TIMESTEP 260452 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.472834 / Loss  0.03367326408624649\n",
      "fps: 7.56109186223009\n",
      "TIMESTEP 260453 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.857546 / Loss  0.020693007856607437\n",
      "fps: 8.212065026069556\n",
      "TIMESTEP 260454 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.975682 / Loss  0.01336289756000042\n",
      "fps: 7.656731648633699\n",
      "TIMESTEP 260455 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.050813 / Loss  0.025749988853931427\n",
      "fps: 7.7415534772633725\n",
      "TIMESTEP 260456 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.938386 / Loss  0.01653381809592247\n",
      "fps: 8.95753700535833\n",
      "TIMESTEP 260457 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.989903 / Loss  0.009777042083442211\n",
      "fps: 7.695871796142068\n",
      "TIMESTEP 260458 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  10.961643 / Loss  0.011456841602921486\n",
      "fps: 8.534739815114389\n",
      "TIMESTEP 260459 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.0315485 / Loss  0.03656663000583649\n",
      "fps: 9.245968133040588\n",
      "TIMESTEP 260460 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.386033 / Loss  0.008641621097922325\n",
      "fps: 9.023229798164506\n",
      "TIMESTEP 260461 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.00375 / Loss  0.009102614596486092\n",
      "fps: 7.742939740886921\n",
      "TIMESTEP 260462 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.322087 / Loss  0.01594718173146248\n",
      "fps: 6.12571855867645\n",
      "TIMESTEP 260463 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.054455 / Loss  0.04194800183176994\n",
      "fps: 9.033374110511897\n",
      "TIMESTEP 260464 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.167303 / Loss  0.012933248654007912\n",
      "fps: 7.585418388513825\n",
      "TIMESTEP 260465 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.121985 / Loss  0.014037754386663437\n",
      "fps: 8.868162769947691\n",
      "TIMESTEP 260466 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.401041 / Loss  0.012704070657491684\n",
      "fps: 8.655473560827431\n",
      "TIMESTEP 260467 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.206761 / Loss  0.027678046375513077\n",
      "fps: 8.817639045392623\n",
      "TIMESTEP 260468 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.044968 / Loss  0.022809568792581558\n",
      "fps: 8.992586049423478\n",
      "TIMESTEP 260469 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.165776 / Loss  0.03516920655965805\n",
      "fps: 8.97133189169303\n",
      "TIMESTEP 260470 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.329389 / Loss  0.026393409818410873\n",
      "fps: 8.16999526665485\n",
      "TIMESTEP 260471 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  10.979692 / Loss  0.0195467472076416\n",
      "fps: 8.982802414087029\n",
      "TIMESTEP 260472 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.1895 / Loss  0.02366282045841217\n",
      "fps: 8.808343168673636\n",
      "TIMESTEP 260473 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.260105 / Loss  0.028067225590348244\n",
      "fps: 7.681903596716471\n",
      "TIMESTEP 260474 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.424962 / Loss  0.03620268777012825\n",
      "fps: 9.120153774565443\n",
      "TIMESTEP 260475 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.386349 / Loss  0.01823096163570881\n",
      "fps: 9.008114017190204\n",
      "TIMESTEP 260476 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.236966 / Loss  0.02334975078701973\n",
      "fps: 8.653812745395387\n",
      "TIMESTEP 260477 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.599918 / Loss  0.02637198008596897\n",
      "fps: 9.330856556820887\n",
      "TIMESTEP 260478 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.632771 / Loss  0.014135674573481083\n",
      "fps: 8.946207202518577\n",
      "TIMESTEP 260479 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.801383 / Loss  0.01812661811709404\n",
      "fps: 8.980359830683028\n",
      "TIMESTEP 260480 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.921616 / Loss  0.024183277040719986\n",
      "fps: 8.896811676590184\n",
      "TIMESTEP 260481 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.6357975 / Loss  0.02034861594438553\n",
      "fps: 8.977534246575342\n",
      "TIMESTEP 260482 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.26092 / Loss  0.022559914737939835\n",
      "fps: 8.59971377371721\n",
      "TIMESTEP 260483 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.920656 / Loss  0.031178247183561325\n",
      "fps: 9.234974987669979\n",
      "TIMESTEP 260484 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.648571 / Loss  0.03080696612596512\n",
      "fps: 8.835470769645596\n",
      "TIMESTEP 260485 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.108477 / Loss  0.029578540474176407\n",
      "fps: 8.873115604466276\n",
      "TIMESTEP 260486 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.78881 / Loss  0.009472602978348732\n",
      "fps: 7.7778016785779185\n",
      "TIMESTEP 260487 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.933988 / Loss  0.01598285511136055\n",
      "fps: 7.876863905769938\n",
      "TIMESTEP 260488 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.913595 / Loss  0.02435244433581829\n",
      "fps: 8.453292594936453\n",
      "TIMESTEP 260489 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.03801 / Loss  0.017916394397616386\n",
      "fps: 9.328428516144529\n",
      "TIMESTEP 260490 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.865552 / Loss  0.02270878106355667\n",
      "fps: 8.706788893317391\n",
      "TIMESTEP 260491 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.114358 / Loss  0.021633917465806007\n",
      "fps: 8.913752834491216\n",
      "TIMESTEP 260492 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.940286 / Loss  0.03243674337863922\n",
      "fps: 8.929750904832872\n",
      "TIMESTEP 260493 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.985156 / Loss  0.020774491131305695\n",
      "fps: 8.970372540758342\n",
      "TIMESTEP 260494 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.179381 / Loss  0.02255685068666935\n",
      "fps: 7.899219740214736\n",
      "TIMESTEP 260495 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.85722 / Loss  0.014328798279166222\n",
      "fps: 7.727376401337546\n",
      "TIMESTEP 260496 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.947215 / Loss  0.02559627965092659\n",
      "fps: 8.58273223775351\n",
      "TIMESTEP 260497 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.000642 / Loss  0.03161612153053284\n",
      "fps: 7.740481964175645\n",
      "TIMESTEP 260498 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.891526 / Loss  0.14607101678848267\n",
      "fps: 5.788640237380533\n",
      "TIMESTEP 260499 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.426866 / Loss  0.03059307299554348\n",
      "fps: 9.003415211105553\n",
      "TIMESTEP 260500 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.000644 / Loss  0.01653294265270233\n",
      "fps: 8.486388164430569\n",
      "TIMESTEP 260501 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.6572075 / Loss  0.8971126675605774\n",
      "fps: 8.913298212581843\n",
      "TIMESTEP 260502 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.022588 / Loss  0.01686648279428482\n",
      "fps: 7.428700974835638\n",
      "TIMESTEP 260503 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.193205 / Loss  0.03549528121948242\n",
      "fps: 8.732693592949392\n",
      "TIMESTEP 260504 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.083873 / Loss  0.037825312465429306\n",
      "fps: 8.432134679214103\n",
      "TIMESTEP 260505 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.040105 / Loss  0.03685152530670166\n",
      "fps: 8.585455319567034\n",
      "TIMESTEP 260506 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.278105 / Loss  0.029483117163181305\n",
      "fps: 8.41911888684593\n",
      "TIMESTEP 260507 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.638587 / Loss  0.025110792368650436\n",
      "fps: 8.9516679116423\n",
      "TIMESTEP 260508 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.312568 / Loss  0.024464115500450134\n",
      "fps: 8.782558441449666\n",
      "TIMESTEP 260509 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.174879 / Loss  0.026013899594545364\n",
      "fps: 8.716722882752572\n",
      "TIMESTEP 260510 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.199045 / Loss  0.013141126371920109\n",
      "fps: 8.774474280980394\n",
      "TIMESTEP 260511 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.040967 / Loss  0.02448420785367489\n",
      "fps: 8.881533086289043\n",
      "TIMESTEP 260512 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.535813 / Loss  0.027516763657331467\n",
      "fps: 8.207822501580194\n",
      "TIMESTEP 260513 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.313296 / Loss  0.0382404550909996\n",
      "fps: 7.759025178931167\n",
      "TIMESTEP 260514 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.465439 / Loss  0.05247081443667412\n",
      "fps: 7.520043030031376\n",
      "TIMESTEP 260515 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.255889 / Loss  0.023111708462238312\n",
      "fps: 7.293363589730213\n",
      "TIMESTEP 260516 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.604131 / Loss  0.03116801008582115\n",
      "fps: 8.500732663835997\n",
      "TIMESTEP 260517 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.780304 / Loss  0.014999063685536385\n",
      "fps: 8.939590735384702\n",
      "TIMESTEP 260518 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.515979 / Loss  0.1634921133518219\n",
      "fps: 8.199078110399993\n",
      "TIMESTEP 260519 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.51869 / Loss  0.026683468371629715\n",
      "fps: 8.755736010938659\n",
      "TIMESTEP 260520 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.50582 / Loss  0.04870498180389404\n",
      "fps: 8.895207879132345\n",
      "TIMESTEP 260521 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.671297 / Loss  0.03312678635120392\n",
      "fps: 8.620906958914922\n",
      "TIMESTEP 260522 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.58649 / Loss  0.02193128503859043\n",
      "fps: 7.483587793750568\n",
      "TIMESTEP 260523 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.720046 / Loss  0.24074898660182953\n",
      "fps: 9.020047398053325\n",
      "TIMESTEP 260524 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.4068575 / Loss  0.01961107924580574\n",
      "fps: 8.485014737446871\n",
      "TIMESTEP 260525 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.444397 / Loss  0.011897409334778786\n",
      "fps: 8.976093306939168\n",
      "TIMESTEP 260526 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.567891 / Loss  0.03794685751199722\n",
      "fps: 8.777540834370978\n",
      "TIMESTEP 260527 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.442257 / Loss  0.044179294258356094\n",
      "fps: 8.482698087183035\n",
      "TIMESTEP 260528 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.471692 / Loss  0.008624980226159096\n",
      "fps: 8.386930613877224\n",
      "TIMESTEP 260529 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.823586 / Loss  0.05372663959860802\n",
      "fps: 7.8510909118135945\n",
      "TIMESTEP 260530 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.67798 / Loss  0.024383950978517532\n",
      "fps: 6.090768232135259\n",
      "TIMESTEP 260531 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.560077 / Loss  0.012421991676092148\n",
      "fps: 7.707496531510424\n",
      "TIMESTEP 260532 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.294042 / Loss  0.023018721491098404\n",
      "fps: 8.43305017441919\n",
      "TIMESTEP 260533 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.290798 / Loss  0.040272124111652374\n",
      "fps: 7.4665356468303345\n",
      "TIMESTEP 260534 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.432514 / Loss  0.059684380888938904\n",
      "fps: 5.563777841744941\n",
      "TIMESTEP 260535 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.549764 / Loss  0.46826955676078796\n",
      "fps: 8.891643859719787\n",
      "TIMESTEP 260536 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.501027 / Loss  0.0403989776968956\n",
      "fps: 8.436815966432194\n",
      "TIMESTEP 260537 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.58709 / Loss  0.1245148777961731\n",
      "fps: 8.967821665747286\n",
      "TIMESTEP 260538 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.705376 / Loss  0.018370909616351128\n",
      "fps: 8.717501912141266\n",
      "TIMESTEP 260539 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.6602 / Loss  0.028833307325839996\n",
      "fps: 8.474522711246617\n",
      "TIMESTEP 260540 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.557155 / Loss  0.021553885191679\n",
      "fps: 8.647675049121585\n",
      "TIMESTEP 260541 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.521945 / Loss  0.009535114280879498\n",
      "fps: 7.6700753419647425\n",
      "TIMESTEP 260542 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.687893 / Loss  0.0049987537786364555\n",
      "fps: 6.987048077868122\n",
      "TIMESTEP 260543 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.278631 / Loss  0.016306815668940544\n",
      "fps: 7.5085328631143655\n",
      "TIMESTEP 260544 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.837638 / Loss  0.01322336494922638\n",
      "fps: 7.216354508260183\n",
      "TIMESTEP 260545 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.372018 / Loss  0.06876294314861298\n",
      "fps: 8.086886273148817\n",
      "TIMESTEP 260546 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.397818 / Loss  0.01765231229364872\n",
      "fps: 7.56086015225178\n",
      "TIMESTEP 260547 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.163999 / Loss  0.026681851595640182\n",
      "fps: 8.389262362464272\n",
      "TIMESTEP 260548 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.586487 / Loss  0.016881577670574188\n",
      "fps: 7.182310262219231\n",
      "TIMESTEP 260549 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.477715 / Loss  0.02252206951379776\n",
      "fps: 7.168206505949146\n",
      "TIMESTEP 260550 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.002752 / Loss  0.02181132137775421\n",
      "fps: 7.3957441406112245\n",
      "TIMESTEP 260551 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.370701 / Loss  0.010032311081886292\n",
      "fps: 7.839073284602776\n",
      "TIMESTEP 260552 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.189122 / Loss  0.02528606727719307\n",
      "fps: 8.185573046162974\n",
      "TIMESTEP 260553 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.879357 / Loss  0.010429794900119305\n",
      "fps: 8.275808727050304\n",
      "TIMESTEP 260554 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.010242 / Loss  0.009863767772912979\n",
      "fps: 8.164746967641213\n",
      "TIMESTEP 260555 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.197617 / Loss  0.022270485758781433\n",
      "fps: 8.360150049232413\n",
      "TIMESTEP 260556 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.463515 / Loss  0.027755647897720337\n",
      "fps: 7.5815518098422885\n",
      "TIMESTEP 260557 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.486743 / Loss  0.018226835876703262\n",
      "fps: 7.410103794002032\n",
      "TIMESTEP 260558 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.338442 / Loss  0.0253303125500679\n",
      "fps: 8.65883285576264\n",
      "TIMESTEP 260559 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.277056 / Loss  0.033474426716566086\n",
      "fps: 8.313800423786766\n",
      "TIMESTEP 260560 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.604917 / Loss  0.044535763561725616\n",
      "fps: 8.514175053691847\n",
      "TIMESTEP 260561 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.473373 / Loss  0.05281012877821922\n",
      "fps: 8.586791139499653\n",
      "TIMESTEP 260562 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.440404 / Loss  0.05568745732307434\n",
      "fps: 8.827047446676417\n",
      "TIMESTEP 260563 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.493457 / Loss  0.04107913002371788\n",
      "fps: 6.915154532774533\n",
      "TIMESTEP 260564 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.462569 / Loss  0.8293900489807129\n",
      "fps: 8.586404412035654\n",
      "TIMESTEP 260565 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.363383 / Loss  0.021571019664406776\n",
      "fps: 8.393576997582569\n",
      "TIMESTEP 260566 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.338917 / Loss  0.013136560097336769\n",
      "fps: 7.6894663413149615\n",
      "TIMESTEP 260567 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.6525545 / Loss  0.015990963205695152\n",
      "fps: 8.203054898202655\n",
      "TIMESTEP 260568 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.737249 / Loss  0.021935805678367615\n",
      "fps: 8.456854818384361\n",
      "TIMESTEP 260569 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.432098 / Loss  0.013162490911781788\n",
      "fps: 7.811218361178414\n",
      "TIMESTEP 260570 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.517966 / Loss  0.012181936763226986\n",
      "fps: 5.221634474730939\n",
      "TIMESTEP 260571 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.881676 / Loss  0.04017748311161995\n",
      "fps: 8.618090514212449\n",
      "TIMESTEP 260572 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.53603 / Loss  0.032238371670246124\n",
      "fps: 7.1126667175404235\n",
      "TIMESTEP 260573 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.896568 / Loss  0.1803734004497528\n",
      "fps: 7.246438807966013\n",
      "TIMESTEP 260574 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.336138 / Loss  0.012752876617014408\n",
      "fps: 7.248104293248726\n",
      "TIMESTEP 260575 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.48647 / Loss  0.02727755904197693\n",
      "fps: 8.390521038796846\n",
      "TIMESTEP 260576 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.246516 / Loss  0.029841043055057526\n",
      "fps: 7.37768771811325\n",
      "TIMESTEP 260577 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.936766 / Loss  0.02079414203763008\n",
      "fps: 7.47483884994912\n",
      "TIMESTEP 260578 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.463836 / Loss  0.028299696743488312\n",
      "fps: 7.06165281881624\n",
      "TIMESTEP 260579 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.2419615 / Loss  0.015264892019331455\n",
      "fps: 7.357743929073262\n",
      "TIMESTEP 260580 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.006954 / Loss  0.027653975412249565\n",
      "fps: 8.404138848592199\n",
      "TIMESTEP 260581 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.997627 / Loss  0.04021105170249939\n",
      "fps: 8.25279992444336\n",
      "TIMESTEP 260582 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.910726 / Loss  0.025971794500947\n",
      "fps: 8.420910613111896\n",
      "TIMESTEP 260583 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.949454 / Loss  0.024337224662303925\n",
      "fps: 8.68427275004814\n",
      "TIMESTEP 260584 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.154067 / Loss  0.00998038612306118\n",
      "fps: 8.31615763468661\n",
      "TIMESTEP 260585 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.938682 / Loss  0.20866480469703674\n",
      "fps: 7.568650661893145\n",
      "TIMESTEP 260586 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.16179 / Loss  0.0260632261633873\n",
      "fps: 8.166273049942271\n",
      "TIMESTEP 260587 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.89517 / Loss  0.029223499819636345\n",
      "fps: 8.116400171062892\n",
      "TIMESTEP 260588 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.092348 / Loss  0.014549298211932182\n",
      "fps: 8.457229963927306\n",
      "TIMESTEP 260589 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.019693 / Loss  0.012420800514519215\n",
      "fps: 8.596153530848813\n",
      "TIMESTEP 260590 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.971648 / Loss  0.076377272605896\n",
      "fps: 8.145181117133353\n",
      "TIMESTEP 260591 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.115671 / Loss  0.01990291103720665\n",
      "fps: 8.671381759166417\n",
      "TIMESTEP 260592 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.812201 / Loss  0.01552400179207325\n",
      "fps: 8.444867708675194\n",
      "TIMESTEP 260593 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.893297 / Loss  0.015453794039785862\n",
      "fps: 8.29435099528953\n",
      "TIMESTEP 260594 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.858315 / Loss  0.014604284428060055\n",
      "fps: 8.238259759391113\n",
      "TIMESTEP 260595 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.816897 / Loss  0.012502733618021011\n",
      "fps: 7.452287672279463\n",
      "TIMESTEP 260596 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.007645 / Loss  0.009717711247503757\n",
      "fps: 8.085389714911393\n",
      "TIMESTEP 260597 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.154415 / Loss  0.01835600845515728\n",
      "fps: 7.49035022081895\n",
      "TIMESTEP 260598 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.271681 / Loss  0.014571646228432655\n",
      "fps: 8.195313339572834\n",
      "TIMESTEP 260599 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.983574 / Loss  0.013074269518256187\n",
      "fps: 8.256682723444799\n",
      "TIMESTEP 260600 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.094514 / Loss  0.03805660456418991\n",
      "fps: 8.32880716455847\n",
      "TIMESTEP 260601 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.796817 / Loss  0.051082365214824677\n",
      "fps: 8.629172058651074\n",
      "TIMESTEP 260602 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.096604 / Loss  0.022948503494262695\n",
      "fps: 6.933031832772978\n",
      "TIMESTEP 260603 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.011527 / Loss  0.01723645068705082\n",
      "fps: 2.256359605590675\n",
      "TIMESTEP 260604 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.985385 / Loss  0.05230705812573433\n",
      "fps: 5.848230320472063\n",
      "TIMESTEP 260605 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.775876 / Loss  0.044568981975317\n",
      "fps: 5.980107674057852\n",
      "TIMESTEP 260606 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.626752 / Loss  0.0115936528891325\n",
      "fps: 4.918095144290017\n",
      "TIMESTEP 260607 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.033618 / Loss  0.06139235198497772\n",
      "fps: 8.616621025578867\n",
      "TIMESTEP 260608 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.868223 / Loss  0.02506626397371292\n",
      "fps: 8.453940048696325\n",
      "TIMESTEP 260609 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.701962 / Loss  0.006138356868177652\n",
      "fps: 8.78736086115761\n",
      "TIMESTEP 260610 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.662829 / Loss  0.017806461080908775\n",
      "fps: 7.162967036345068\n",
      "TIMESTEP 260611 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.690211 / Loss  0.01301316823810339\n",
      "fps: 8.565905103839265\n",
      "TIMESTEP 260612 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.418376 / Loss  1.011178731918335\n",
      "fps: 7.5754856701127755\n",
      "TIMESTEP 260613 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.664931 / Loss  0.21546229720115662\n",
      "fps: 7.71420163616032\n",
      "TIMESTEP 260614 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.7457075 / Loss  0.019249167293310165\n",
      "fps: 7.388370413429865\n",
      "TIMESTEP 260615 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.286422 / Loss  0.02060609683394432\n",
      "fps: 8.231743888461478\n",
      "TIMESTEP 260616 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.68726 / Loss  1.70772123336792\n",
      "fps: 7.009186147750421\n",
      "TIMESTEP 260617 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.4853 / Loss  0.024547776207327843\n",
      "fps: 7.500691179547523\n",
      "TIMESTEP 260618 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.972493 / Loss  0.013760816305875778\n",
      "fps: 7.968008693124719\n",
      "TIMESTEP 260619 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.651495 / Loss  0.014193391427397728\n",
      "fps: 8.104465441616911\n",
      "TIMESTEP 260620 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.7946825 / Loss  0.010806858539581299\n",
      "fps: 6.975393314485282\n",
      "TIMESTEP 260621 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.864961 / Loss  0.01315249316394329\n",
      "fps: 8.01346562044451\n",
      "TIMESTEP 260622 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.160244 / Loss  2.8879494667053223\n",
      "fps: 8.710803188329947\n",
      "TIMESTEP 260623 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.754652 / Loss  0.04158516973257065\n",
      "fps: 6.5593907121130375\n",
      "TIMESTEP 260624 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.49967 / Loss  0.01357177458703518\n",
      "fps: 7.9992981572814275\n",
      "TIMESTEP 260625 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.459053 / Loss  0.007933452725410461\n",
      "fps: 8.300539278257686\n",
      "TIMESTEP 260626 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.585642 / Loss  0.015972452238202095\n",
      "fps: 7.901094657446252\n",
      "TIMESTEP 260627 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.629697 / Loss  0.0634755939245224\n",
      "fps: 8.119259679395727\n",
      "TIMESTEP 260628 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.717658 / Loss  0.04685697332024574\n",
      "fps: 8.803185203997037\n",
      "TIMESTEP 260629 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.742349 / Loss  0.05992091819643974\n",
      "fps: 6.7636210002144725\n",
      "TIMESTEP 260630 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.652215 / Loss  0.3409993648529053\n",
      "fps: 8.630077570420362\n",
      "TIMESTEP 260631 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.806107 / Loss  0.021356740966439247\n",
      "fps: 8.076437536345185\n",
      "TIMESTEP 260632 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.549877 / Loss  0.015336322598159313\n",
      "fps: 7.822450334584138\n",
      "TIMESTEP 260633 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.722425 / Loss  0.028427474200725555\n",
      "fps: 8.129756551403318\n",
      "TIMESTEP 260634 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.251018 / Loss  0.03133702278137207\n",
      "fps: 8.601636120151676\n",
      "TIMESTEP 260635 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.53168 / Loss  0.1008630245923996\n",
      "fps: 7.560301380727496\n",
      "TIMESTEP 260636 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.720585 / Loss  0.023975834250450134\n",
      "fps: 8.643861221882181\n",
      "TIMESTEP 260637 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.607792 / Loss  0.019770950078964233\n",
      "fps: 8.539709622052598\n",
      "TIMESTEP 260638 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.48027 / Loss  0.024941936135292053\n",
      "fps: 6.860836850197926\n",
      "TIMESTEP 260639 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.557874 / Loss  0.024375125765800476\n",
      "fps: 8.409092549450461\n",
      "TIMESTEP 260640 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.901946 / Loss  0.022601157426834106\n",
      "fps: 8.466089652499678\n",
      "TIMESTEP 260641 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.041006 / Loss  0.025863610208034515\n",
      "fps: 7.721344823079146\n",
      "TIMESTEP 260642 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.326149 / Loss  0.0176396407186985\n",
      "fps: 5.688796297803313\n",
      "TIMESTEP 260643 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.881455 / Loss  0.019006486982107162\n",
      "fps: 8.479885487386225\n",
      "TIMESTEP 260644 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.420767 / Loss  0.01577824354171753\n",
      "fps: 8.468243360562733\n",
      "TIMESTEP 260645 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.14648 / Loss  0.028389401733875275\n",
      "fps: 8.427915799287879\n",
      "TIMESTEP 260646 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.102612 / Loss  0.007484615780413151\n",
      "fps: 5.745865920333603\n",
      "TIMESTEP 260647 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.057075 / Loss  0.025093883275985718\n",
      "fps: 7.5794282409917235\n",
      "TIMESTEP 260648 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.970954 / Loss  0.06676948070526123\n",
      "fps: 8.591065038702052\n",
      "TIMESTEP 260649 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.0057535 / Loss  0.054396308958530426\n",
      "fps: 7.782174286546572\n",
      "TIMESTEP 260650 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.139405 / Loss  0.05783633887767792\n",
      "fps: 8.418730216715408\n",
      "TIMESTEP 260651 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.215493 / Loss  0.03275604546070099\n",
      "fps: 7.959254002110169\n",
      "TIMESTEP 260652 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.933233 / Loss  0.028834471479058266\n",
      "fps: 8.518238554319636\n",
      "TIMESTEP 260653 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.27219 / Loss  0.031178146600723267\n",
      "fps: 7.498049647291852\n",
      "TIMESTEP 260654 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.2018175 / Loss  0.015378402546048164\n",
      "fps: 6.977006159747555\n",
      "TIMESTEP 260655 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.963324 / Loss  0.01657649502158165\n",
      "fps: 8.58266198753824\n",
      "TIMESTEP 260656 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.21316 / Loss  0.18598954379558563\n",
      "fps: 6.658946616392141\n",
      "TIMESTEP 260657 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.2453375 / Loss  0.03674590215086937\n",
      "fps: 7.818003217190159\n",
      "TIMESTEP 260658 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.523052 / Loss  0.02666531689465046\n",
      "fps: 8.506232178565199\n",
      "TIMESTEP 260659 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.904757 / Loss  0.024770256131887436\n",
      "fps: 7.325732170802594\n",
      "TIMESTEP 260660 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.312384 / Loss  0.008146501146256924\n",
      "fps: 8.405351460108536\n",
      "TIMESTEP 260661 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.339748 / Loss  0.20173370838165283\n",
      "fps: 8.701171481975525\n",
      "TIMESTEP 260662 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.177398 / Loss  0.014428408816456795\n",
      "fps: 7.874009953499025\n",
      "TIMESTEP 260663 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.12038 / Loss  0.017957745119929314\n",
      "fps: 7.662942656330787\n",
      "TIMESTEP 260664 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.320764 / Loss  0.018647652119398117\n",
      "fps: 8.303941017386725\n",
      "TIMESTEP 260665 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.550977 / Loss  0.029558910056948662\n",
      "fps: 6.652651747661669\n",
      "TIMESTEP 260666 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.6057415 / Loss  0.1285208910703659\n",
      "fps: 7.34617156288368\n",
      "TIMESTEP 260667 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.645858 / Loss  0.028470439836382866\n",
      "fps: 8.302132982850628\n",
      "TIMESTEP 260668 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.479632 / Loss  0.10036192834377289\n",
      "fps: 7.5441599951076235\n",
      "TIMESTEP 260669 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.220385 / Loss  0.05564180389046669\n",
      "fps: 8.260357686558493\n",
      "TIMESTEP 260670 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.172742 / Loss  0.030039329081773758\n",
      "fps: 8.128307093633481\n",
      "TIMESTEP 260671 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.774742 / Loss  0.021854761987924576\n",
      "fps: 6.811533620780655\n",
      "TIMESTEP 260672 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.576126 / Loss  0.024361366406083107\n",
      "fps: 6.933031832772978\n",
      "TIMESTEP 260673 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.253442 / Loss  0.024608157575130463\n",
      "fps: 6.422028687269565\n",
      "TIMESTEP 260674 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.262617 / Loss  0.032038040459156036\n",
      "fps: 5.954832307096016\n",
      "TIMESTEP 260675 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.859259 / Loss  0.025959504768252373\n",
      "fps: 6.931496526239944\n",
      "TIMESTEP 260676 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.770873 / Loss  0.024582315236330032\n",
      "fps: 6.580250012550831\n",
      "TIMESTEP 260677 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.568469 / Loss  0.04750177264213562\n",
      "fps: 4.980205296873052\n",
      "TIMESTEP 260678 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.929267 / Loss  0.02063535526394844\n",
      "fps: 4.843413423620149\n",
      "TIMESTEP 260679 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.181859 / Loss  0.03818034008145332\n",
      "fps: 7.828889382488376\n",
      "TIMESTEP 260680 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.850164 / Loss  0.2619031071662903\n",
      "fps: 6.680700323657586\n",
      "TIMESTEP 260681 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.9755535 / Loss  0.02144245058298111\n",
      "fps: 7.5654970517623585\n",
      "TIMESTEP 260682 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.724974 / Loss  0.04832277074456215\n",
      "fps: 6.496461590170206\n",
      "TIMESTEP 260683 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.967798 / Loss  0.03648385405540466\n",
      "fps: 6.644084120617662\n",
      "TIMESTEP 260684 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.879377 / Loss  0.02945476397871971\n",
      "fps: 8.229773373884038\n",
      "TIMESTEP 260685 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  13.214792 / Loss  0.02894616313278675\n",
      "fps: 7.338639493156173\n",
      "TIMESTEP 260686 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.828379 / Loss  0.02098444104194641\n",
      "fps: 5.598877635880411\n",
      "TIMESTEP 260687 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.766899 / Loss  0.04551798477768898\n",
      "fps: 6.769209912607022\n",
      "TIMESTEP 260688 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.849738 / Loss  0.04771208018064499\n",
      "fps: 6.212485984409183\n",
      "TIMESTEP 260689 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.912262 / Loss  0.044829223304986954\n",
      "fps: 6.963060602790667\n",
      "TIMESTEP 260690 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.988671 / Loss  0.01779666729271412\n",
      "fps: 6.743687696958004\n",
      "TIMESTEP 260691 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.888939 / Loss  0.03424035757780075\n",
      "fps: 5.824281633735616\n",
      "TIMESTEP 260692 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.781656 / Loss  0.047799430787563324\n",
      "fps: 7.047710584242099\n",
      "TIMESTEP 260693 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.6704855 / Loss  0.026463545858860016\n",
      "fps: 7.553330499989195\n",
      "TIMESTEP 260694 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.532648 / Loss  0.03439471498131752\n",
      "fps: 7.317220564888959\n",
      "TIMESTEP 260695 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.947443 / Loss  5.620957851409912\n",
      "fps: 7.1627713168365865\n",
      "TIMESTEP 260696 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.536535 / Loss  0.012906698510050774\n",
      "fps: 5.202592424909606\n",
      "TIMESTEP 260697 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.146104 / Loss  0.061087679117918015\n",
      "fps: 6.574185418090395\n",
      "TIMESTEP 260698 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.681974 / Loss  0.020403727889060974\n",
      "fps: 5.2610767492063175\n",
      "TIMESTEP 260699 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.943487 / Loss  0.019670633599162102\n",
      "fps: 6.058979033464982\n",
      "TIMESTEP 260700 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.889745 / Loss  0.03131069988012314\n",
      "fps: 6.3033851564085035\n",
      "TIMESTEP 260701 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.92712 / Loss  0.11145202070474625\n",
      "fps: 6.547420871552473\n",
      "TIMESTEP 260702 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.196723 / Loss  0.05835374444723129\n",
      "fps: 6.5999647523855085\n",
      "TIMESTEP 260703 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.792651 / Loss  0.02092139981687069\n",
      "fps: 6.452030919509287\n",
      "TIMESTEP 260704 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.549661 / Loss  0.044335249811410904\n",
      "fps: 6.447746142624138\n",
      "TIMESTEP 260705 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  11.596135 / Loss  0.06544624269008636\n",
      "fps: 6.8975053034912595\n",
      "TIMESTEP 260706 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.859818 / Loss  0.04316770285367966\n",
      "fps: 7.1909421954614965\n",
      "TIMESTEP 260707 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.406663 / Loss  0.026443902403116226\n",
      "fps: 7.17565040657361\n",
      "TIMESTEP 260708 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.029975 / Loss  0.16962659358978271\n",
      "fps: 8.452713378260729\n",
      "TIMESTEP 260709 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.528117 / Loss  0.033396653831005096\n",
      "fps: 6.554931470454936\n",
      "TIMESTEP 260710 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.311938 / Loss  0.031760841608047485\n",
      "fps: 6.672537989665793\n",
      "TIMESTEP 260711 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.389848 / Loss  0.04057060182094574\n",
      "fps: 6.881876250059478\n",
      "TIMESTEP 260712 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.664574 / Loss  0.05108216777443886\n",
      "fps: 7.140152359875729\n",
      "TIMESTEP 260713 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.459148 / Loss  0.05441026762127876\n",
      "fps: 7.975948382581974\n",
      "TIMESTEP 260714 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.071741 / Loss  0.014593834057450294\n",
      "fps: 5.089785611051582\n",
      "TIMESTEP 260715 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.732111 / Loss  0.10394394397735596\n",
      "fps: 8.148029287264237\n",
      "TIMESTEP 260716 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.591878 / Loss  0.038038481026887894\n",
      "fps: 7.7869999572989945\n",
      "TIMESTEP 260717 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.85328 / Loss  0.04200737923383713\n",
      "fps: 8.261057275663752\n",
      "TIMESTEP 260718 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.132434 / Loss  0.052922382950782776\n",
      "fps: 7.693909521485987\n",
      "TIMESTEP 260719 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.166741 / Loss  0.02959415875375271\n",
      "fps: 7.909512979788265\n",
      "TIMESTEP 260720 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.966067 / Loss  0.0313766673207283\n",
      "fps: 4.8005127500801175\n",
      "TIMESTEP 260721 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.906043 / Loss  0.036515552550554276\n",
      "fps: 7.200991652674338\n",
      "TIMESTEP 260722 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.775722 / Loss  0.035486094653606415\n",
      "fps: 6.463036698265554\n",
      "TIMESTEP 260723 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.368259 / Loss  0.02798883616924286\n",
      "fps: 7.817813780274592\n",
      "TIMESTEP 260724 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.70353 / Loss  0.02918614260852337\n",
      "fps: 6.652208598395275\n",
      "TIMESTEP 260725 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.94276 / Loss  0.03800608962774277\n",
      "fps: 7.727974381981896\n",
      "TIMESTEP 260726 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.696732 / Loss  0.020059077069163322\n",
      "fps: 7.685225427933788\n",
      "TIMESTEP 260727 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.530491 / Loss  0.04018677398562431\n",
      "fps: 7.737297750000462\n",
      "TIMESTEP 260728 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.873045 / Loss  2.154344081878662\n",
      "fps: 7.352765151050331\n",
      "TIMESTEP 260729 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.9261875 / Loss  0.01953626051545143\n",
      "fps: 8.212885110182535\n",
      "TIMESTEP 260730 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.354226 / Loss  0.08575917780399323\n",
      "fps: 6.708514269354357\n",
      "TIMESTEP 260731 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 1 / REWARD 0.1 / Q_MAX  12.991437 / Loss  0.07789241522550583\n",
      "fps: 7.726251878923635\n",
      "TIMESTEP 260732 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  13.036069 / Loss  0.021271804347634315\n",
      "fps: 7.685676066919539\n",
      "TIMESTEP 260733 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.793076 / Loss  0.08593101799488068\n",
      "fps: 7.832866146878939\n",
      "TIMESTEP 260734 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.396725 / Loss  0.27583545446395874\n",
      "fps: 7.778335364010466\n",
      "TIMESTEP 260735 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.863111 / Loss  0.02684842422604561\n",
      "fps: 7.243060100608035\n",
      "TIMESTEP 260736 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.76244 / Loss  0.04784560576081276\n",
      "fps: 8.37310101691667\n",
      "TIMESTEP 260737 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.417312 / Loss  0.017315231263637543\n",
      "fps: 8.160727835910048\n",
      "TIMESTEP 260738 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.7748165 / Loss  0.025846723467111588\n",
      "fps: 8.422533966818344\n",
      "TIMESTEP 260739 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.851195 / Loss  0.4147792160511017\n",
      "fps: 8.015762808715412\n",
      "TIMESTEP 260740 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  13.239754 / Loss  0.11379135400056839\n",
      "fps: 8.236415514129908\n",
      "TIMESTEP 260741 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.875228 / Loss  2.038099527359009\n",
      "fps: 8.049328791440772\n",
      "TIMESTEP 260742 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.905124 / Loss  0.04421025887131691\n",
      "fps: 8.316701796269454\n",
      "TIMESTEP 260743 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.431669 / Loss  0.03718344494700432\n",
      "fps: 7.913258558397087\n",
      "TIMESTEP 260744 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.799694 / Loss  0.1812880039215088\n",
      "fps: 8.298962605930736\n",
      "TIMESTEP 260745 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.470018 / Loss  0.023462096229195595\n",
      "fps: 7.146028057286751\n",
      "TIMESTEP 260746 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.588632 / Loss  0.04008615016937256\n",
      "fps: 6.616028430770978\n",
      "TIMESTEP 260747 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.651829 / Loss  0.19304072856903076\n",
      "fps: 7.728686384383067\n",
      "TIMESTEP 260748 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.006559 / Loss  0.036528393626213074\n",
      "fps: 7.738097126927697\n",
      "TIMESTEP 260749 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.376353 / Loss  0.08945949375629425\n",
      "fps: 7.7538040614861305\n",
      "TIMESTEP 260750 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.559159 / Loss  0.03351834416389465\n",
      "fps: 5.31883967916812\n",
      "TIMESTEP 260751 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.0805 / Loss  0.058563701808452606\n",
      "fps: 8.44843985102496\n",
      "TIMESTEP 260752 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.9079075 / Loss  0.03545161336660385\n",
      "fps: 8.156728521588288\n",
      "TIMESTEP 260753 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.848779 / Loss  0.00870836153626442\n",
      "fps: 8.009349292310512\n",
      "TIMESTEP 260754 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.23871 / Loss  0.06464144587516785\n",
      "fps: 7.593479567525292\n",
      "TIMESTEP 260755 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.084015 / Loss  0.024626312777400017\n",
      "fps: 7.681903596716471\n",
      "TIMESTEP 260756 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.255766 / Loss  3.1664140224456787\n",
      "fps: 7.358789554574611\n",
      "TIMESTEP 260757 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.152427 / Loss  0.1517375111579895\n",
      "fps: 7.822289858802423\n",
      "TIMESTEP 260758 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.017296 / Loss  0.08648964762687683\n",
      "fps: 7.6024672695272635\n",
      "TIMESTEP 260759 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.728377 / Loss  0.013775882311165333\n",
      "fps: 8.026715504752719\n",
      "TIMESTEP 260760 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.187068 / Loss  0.0178816094994545\n",
      "fps: 7.696690883000061\n",
      "TIMESTEP 260761 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.756358 / Loss  0.02749437838792801\n",
      "fps: 8.326095673681996\n",
      "TIMESTEP 260762 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.837476 / Loss  0.026207484304904938\n",
      "fps: 8.396164548093283\n",
      "TIMESTEP 260763 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.725264 / Loss  0.015160384587943554\n",
      "fps: 8.590326035976807\n",
      "TIMESTEP 260764 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.614413 / Loss  0.026730496436357498\n",
      "fps: 8.107081488239398\n",
      "TIMESTEP 260765 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.914753 / Loss  0.039270635694265366\n",
      "fps: 8.274861750379287\n",
      "TIMESTEP 260766 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.041111 / Loss  4.411259174346924\n",
      "fps: 8.112350032009934\n",
      "TIMESTEP 260767 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.808688 / Loss  0.028691699728369713\n",
      "fps: 7.989698341219588\n",
      "TIMESTEP 260768 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.603296 / Loss  0.02460925467312336\n",
      "fps: 7.912273321499111\n",
      "TIMESTEP 260769 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.332728 / Loss  0.17270037531852722\n",
      "fps: 8.579343975257935\n",
      "TIMESTEP 260770 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.116779 / Loss  0.01863677427172661\n",
      "fps: 8.379909653779377\n",
      "TIMESTEP 260771 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.601051 / Loss  0.020648645237088203\n",
      "fps: 8.568477479244295\n",
      "TIMESTEP 260772 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.420909 / Loss  0.02193485014140606\n",
      "fps: 8.31366859131786\n",
      "TIMESTEP 260773 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.202886 / Loss  0.04783635586500168\n",
      "fps: 7.330405345180441\n",
      "TIMESTEP 260774 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.442094 / Loss  0.022145526483654976\n",
      "fps: 8.013159428153305\n",
      "TIMESTEP 260775 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.596783 / Loss  0.04283953085541725\n",
      "fps: 8.34843532919591\n",
      "TIMESTEP 260776 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.612584 / Loss  0.025629382580518723\n",
      "fps: 8.117609717200963\n",
      "TIMESTEP 260777 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.149639 / Loss  0.026684369891881943\n",
      "fps: 8.559245110523394\n",
      "TIMESTEP 260778 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.677536 / Loss  0.012250801548361778\n",
      "fps: 8.4287287488445\n",
      "TIMESTEP 260779 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.603076 / Loss  0.016793500632047653\n",
      "fps: 7.665071263183095\n",
      "TIMESTEP 260780 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.65614 / Loss  0.07215636223554611\n",
      "fps: 8.575888247549997\n",
      "TIMESTEP 260781 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.612742 / Loss  0.01642823964357376\n",
      "fps: 8.82723321863392\n",
      "TIMESTEP 260782 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.684734 / Loss  0.03456805646419525\n",
      "fps: 8.418848503825743\n",
      "TIMESTEP 260783 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.263666 / Loss  4.230529308319092\n",
      "fps: 8.923462495026923\n",
      "TIMESTEP 260784 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.618704 / Loss  0.015558384358882904\n",
      "fps: 8.56391126356518\n",
      "TIMESTEP 260785 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.499565 / Loss  0.007515073753893375\n",
      "fps: 8.526377202843546\n",
      "TIMESTEP 260786 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.230462 / Loss  0.02516062930226326\n",
      "fps: 5.932352882937165\n",
      "TIMESTEP 260787 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.492329 / Loss  0.023934077471494675\n",
      "fps: 8.77267574407563\n",
      "TIMESTEP 260788 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.295616 / Loss  0.3192474842071533\n",
      "fps: 8.364084504233587\n",
      "TIMESTEP 260789 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.545104 / Loss  0.012484634295105934\n",
      "fps: 8.968051760339025\n",
      "TIMESTEP 260790 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.777885 / Loss  0.4768519103527069\n",
      "fps: 8.820605999427984\n",
      "TIMESTEP 260791 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.872337 / Loss  0.019025903195142746\n",
      "fps: 8.721507958786896\n",
      "TIMESTEP 260792 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.756424 / Loss  0.028207845985889435\n",
      "fps: 8.816471354850915\n",
      "TIMESTEP 260793 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.919046 / Loss  0.032599933445453644\n",
      "fps: 8.796852322586599\n",
      "TIMESTEP 260794 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.844589 / Loss  0.014540590345859528\n",
      "fps: 8.145908793409168\n",
      "TIMESTEP 260795 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.1810255 / Loss  0.012467880733311176\n",
      "fps: 8.534531418188182\n",
      "TIMESTEP 260796 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.872421 / Loss  0.03513457626104355\n",
      "fps: 8.92063465055936\n",
      "TIMESTEP 260797 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.767943 / Loss  0.020392989739775658\n",
      "fps: 8.755571513263916\n",
      "TIMESTEP 260798 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.276628 / Loss  0.021366849541664124\n",
      "fps: 8.67574030100196\n",
      "TIMESTEP 260799 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.2711 / Loss  0.17834514379501343\n",
      "fps: 8.880198000935804\n",
      "TIMESTEP 260800 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.631588 / Loss  0.01927429810166359\n",
      "fps: 8.496961249858192\n",
      "TIMESTEP 260801 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.089584 / Loss  0.161875918507576\n",
      "fps: 8.861324200816771\n",
      "TIMESTEP 260802 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.163144 / Loss  0.021035078912973404\n",
      "fps: 8.627521320404645\n",
      "TIMESTEP 260803 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.766948 / Loss  0.01442425325512886\n",
      "fps: 8.694137375655277\n",
      "TIMESTEP 260804 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.225314 / Loss  0.02851148694753647\n",
      "fps: 8.566254996109315\n",
      "TIMESTEP 260805 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.3671255 / Loss  0.022220056504011154\n",
      "fps: 7.223600205979963\n",
      "TIMESTEP 260806 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.05191 / Loss  0.05330941826105118\n",
      "fps: 8.244024301795116\n",
      "TIMESTEP 260807 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.566579 / Loss  0.02965041995048523\n",
      "fps: 8.78688222367007\n",
      "TIMESTEP 260808 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.114293 / Loss  0.024147111922502518\n",
      "fps: 8.357717874734979\n",
      "TIMESTEP 260809 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.022078 / Loss  0.03782379627227783\n",
      "fps: 8.186307957748129\n",
      "TIMESTEP 260810 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.41194 / Loss  0.02369675412774086\n",
      "fps: 8.13920643637655\n",
      "TIMESTEP 260811 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.273085 / Loss  0.01801154762506485\n",
      "fps: 7.7646989129468915\n",
      "TIMESTEP 260812 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.010412 / Loss  0.03368386998772621\n",
      "fps: 8.292694693178696\n",
      "TIMESTEP 260813 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.66177 / Loss  2.589096784591675\n",
      "fps: 9.163631279057574\n",
      "TIMESTEP 260814 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.090863 / Loss  0.08192434906959534\n",
      "fps: 8.964831510146155\n",
      "TIMESTEP 260815 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.872404 / Loss  0.033669374883174896\n",
      "fps: 8.613435903965302\n",
      "TIMESTEP 260816 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.012077 / Loss  0.007449738681316376\n",
      "fps: 8.705722653371717\n",
      "TIMESTEP 260817 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.850074 / Loss  0.017625072970986366\n",
      "fps: 9.26191385763655\n",
      "TIMESTEP 260818 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.537767 / Loss  0.10874054580926895\n",
      "fps: 8.119401135936878\n",
      "TIMESTEP 260819 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.224546 / Loss  0.04079608619213104\n",
      "fps: 8.578220996915828\n",
      "TIMESTEP 260820 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.185064 / Loss  0.027927540242671967\n",
      "fps: 8.288433986835111\n",
      "TIMESTEP 260821 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.894302 / Loss  0.07473232597112656\n",
      "fps: 7.580702239524876\n",
      "TIMESTEP 260822 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.292791 / Loss  0.02964359149336815\n",
      "fps: 5.255993704292209\n",
      "TIMESTEP 260823 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.115137 / Loss  0.04959314316511154\n",
      "fps: 9.07300274290263\n",
      "TIMESTEP 260824 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.214897 / Loss  0.21478989720344543\n",
      "fps: 8.659744603536735\n",
      "TIMESTEP 260825 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.941889 / Loss  1.7924913167953491\n",
      "fps: 8.870919915823313\n",
      "TIMESTEP 260826 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.220428 / Loss  0.023934461176395416\n",
      "fps: 8.686898728540628\n",
      "TIMESTEP 260827 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.24163 / Loss  0.03594371676445007\n",
      "fps: 8.632670870860983\n",
      "TIMESTEP 260828 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.34703 / Loss  0.01461887639015913\n",
      "fps: 8.931747569714329\n",
      "TIMESTEP 260829 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.101248 / Loss  0.04470844566822052\n",
      "fps: 8.724573891410433\n",
      "TIMESTEP 260830 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.796467 / Loss  0.11865317076444626\n",
      "fps: 8.462400129125978\n",
      "TIMESTEP 260831 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.900523 / Loss  0.024204691872000694\n",
      "fps: 8.864058362444815\n",
      "TIMESTEP 260832 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.032866 / Loss  0.01922934502363205\n",
      "fps: 8.59131140119991\n",
      "TIMESTEP 260833 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.2241125 / Loss  0.012735049240291119\n",
      "fps: 8.900379840848807\n",
      "TIMESTEP 260834 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.802113 / Loss  0.012661338783800602\n",
      "fps: 8.71400642803571\n",
      "TIMESTEP 260835 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.564002 / Loss  0.05566316470503807\n",
      "fps: 9.028687792350926\n",
      "TIMESTEP 260836 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.930253 / Loss  0.06780518591403961\n",
      "fps: 8.534236206629933\n",
      "TIMESTEP 260837 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.715748 / Loss  0.23425325751304626\n",
      "fps: 9.282123801367211\n",
      "TIMESTEP 260838 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.073564 / Loss  0.029091257601976395\n",
      "fps: 8.828868349096021\n",
      "TIMESTEP 260839 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.910382 / Loss  0.013429337181150913\n",
      "fps: 8.691489008985114\n",
      "TIMESTEP 260840 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.177921 / Loss  0.0386355035007\n",
      "fps: 9.172970348565105\n",
      "TIMESTEP 260841 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.099643 / Loss  0.013742130249738693\n",
      "fps: 9.163451097939575\n",
      "TIMESTEP 260842 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.744613 / Loss  0.03198959305882454\n",
      "fps: 8.195777741302631\n",
      "TIMESTEP 260843 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.09602 / Loss  0.016185693442821503\n",
      "fps: 9.222202922131462\n",
      "TIMESTEP 260844 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.99857 / Loss  0.02054246887564659\n",
      "fps: 8.883056702155372\n",
      "TIMESTEP 260845 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.866179 / Loss  0.03015957772731781\n",
      "fps: 8.71915101497781\n",
      "TIMESTEP 260846 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.320278 / Loss  0.02273239567875862\n",
      "fps: 8.73329363311165\n",
      "TIMESTEP 260847 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.453127 / Loss  0.03758695721626282\n",
      "fps: 8.816415758253493\n",
      "TIMESTEP 260848 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.734406 / Loss  0.046652328222990036\n",
      "fps: 8.555473964197713\n",
      "TIMESTEP 260849 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.1457615 / Loss  0.17029468715190887\n",
      "fps: 9.292653044042742\n",
      "TIMESTEP 260850 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.072299 / Loss  0.05034113675355911\n",
      "fps: 8.777173469494523\n",
      "TIMESTEP 260851 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.908305 / Loss  0.03683806210756302\n",
      "fps: 8.980129104086155\n",
      "TIMESTEP 260852 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.047459 / Loss  0.06843148171901703\n",
      "fps: 9.071629098050412\n",
      "TIMESTEP 260853 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.448871 / Loss  0.04081983119249344\n",
      "fps: 9.273442820125052\n",
      "TIMESTEP 260854 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.467557 / Loss  0.014206564985215664\n",
      "fps: 8.757655607291253\n",
      "TIMESTEP 260855 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.389836 / Loss  0.053883589804172516\n",
      "fps: 9.406821133087677\n",
      "TIMESTEP 260856 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.121535 / Loss  0.007372512482106686\n",
      "fps: 8.906219675076072\n",
      "TIMESTEP 260857 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.913591 / Loss  0.033597130328416824\n",
      "fps: 8.668227688784821\n",
      "TIMESTEP 260858 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.856685 / Loss  0.017047274857759476\n",
      "fps: 6.270928116809624\n",
      "TIMESTEP 260859 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.994624 / Loss  0.030977191403508186\n",
      "fps: 9.069549798901527\n",
      "TIMESTEP 260860 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  11.77815 / Loss  0.09864912182092667\n",
      "fps: 8.554845762863973\n",
      "TIMESTEP 260861 / STATE train / EPSILON 9.999999987391849e-05 / ACTION 0 / REWARD 0.1 / Q_MAX  12.029499 / Loss  0.015937820076942444\n",
      "fps: 9.277729974363115\n",
      "TIMESTEP 260862"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-52-72cb52f7ef17>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mplay_Game\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobserve\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-51-5ba40716cf16>\u001b[0m in \u001b[0;36mplay_Game\u001b[0;34m(observe)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mq_target\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuild_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mtrain_Network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq_eval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mq_target\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mgame_state\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mobserve\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobserve\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mgame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-50-53c72357d4c6>\u001b[0m in \u001b[0;36mtrain_Network\u001b[0;34m(q_eval, q_target, game_state, observe)\u001b[0m\n\u001b[1;32m    150\u001b[0m             \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"train\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"TIMESTEP\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/ STATE\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/ EPSILON\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/ ACTION\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/ REWARD\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr_t\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/ Q_MAX \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mQ_sa\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/ Loss \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Episode finished!\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/ipykernel/iostream.py\u001b[0m in \u001b[0;36mwrite\u001b[0;34m(self, string)\u001b[0m\n\u001b[1;32m    398\u001b[0m             \u001b[0mis_child\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_master_process\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    399\u001b[0m             \u001b[0;31m# only touch the buffer in the IO thread to avoid races\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 400\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpub_thread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mschedule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    401\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_child\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m                 \u001b[0;31m# newlines imply flush in subprocesses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/ipykernel/iostream.py\u001b[0m in \u001b[0;36mschedule\u001b[0;34m(self, f)\u001b[0m\n\u001b[1;32m    201\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_events\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0;31m# wake event thread (message content is ignored)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 203\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event_pipe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mb''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    204\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/zmq/sugar/socket.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, data, flags, copy, track, routing_id, group)\u001b[0m\n\u001b[1;32m    393\u001b[0m                                  copy_threshold=self.copy_threshold)\n\u001b[1;32m    394\u001b[0m             \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 395\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSocket\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrack\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    396\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    397\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_multipart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg_parts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrack\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "play_Game(observe=False);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}