Deep Q -  DDQN exponential epsilon decay.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intitializeAgent():\n",
    "    \n",
    "    model = Sequential() \n",
    "    \n",
    "    model.add(Dense(256,activation='relu',input_shape=(INPUT_SIZE,)))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(256,activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(256,activation='relu'))\n",
    "    model.add(Dense(OUTPUT_SIZE,activation='linear'))\n",
    "    \n",
    "    model.compile(optimizer='adam',loss='mse',metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def updateTargetNetwork():\n",
    "    target.set_weights(agent.get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getPrediction(env,agent):\n",
    "    env = env.reshape(INPUT_SIZE,1).reshape(1,-1)\n",
    "    return agent.predict(env)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAction(output,invalidActions):\n",
    "    rand = np.random.rand()\n",
    "    output[invalidActions] = np.min(output) - 0.1\n",
    "    \n",
    "    if rand < EPSILON_RATE ** iteration:\n",
    "        action = np.random.randint(OUTPUT_SIZE)\n",
    "        while action in invalidActions:\n",
    "            action = np.random.randint(OUTPUT_SIZE)\n",
    "        return action\n",
    "    else:\n",
    "        return np.argmax(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getGreedyAction(output,invalidActions):\n",
    "    output[invalidActions] = np.min(output) - 0.1\n",
    "    return np.argmax(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(initialStateMemory,actionMemory,rewardMemory,finalStateMemory,evalnet,targetnet,epochs):\n",
    "    \n",
    "    if memCount < MAX_MEM_SIZE:\n",
    "        s = np.random.choice(memCount, SAMPLE_SIZE)\n",
    "    else:\n",
    "        s = np.random.choice(MAX_MEM_SIZE, SAMPLE_SIZE)\n",
    "    batchSize = s.shape[0] // BATCH_SIZE\n",
    "    \n",
    "    if batchSize == 0:\n",
    "        batchSize = 1\n",
    "                \n",
    "    targetQs = getTargetQs(initialStateMemory[s], actionMemory[s], rewardMemory[s], finalStateMemory[s], evalnet, targetnet)\n",
    "        \n",
    "    print(\"Training Model...\")\n",
    "    evalnet.fit(initialStateMemory[s],targetQs,batch_size = batchSize, epochs = epochs, verbose = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Replay Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addMemory(initialState, action, reward, finalState, memCount, MAX_MEM_SIZE):\n",
    "    if memCount < MAX_MEM_SIZE:\n",
    "        initialStateMemory[memCount] = initialState.reshape(INPUT_SIZE)\n",
    "        actionMemory[memCount] = action\n",
    "        rewardMemory[memCount] = reward\n",
    "        finalStateMemory[memCount] = finalState.reshape(INPUT_SIZE)\n",
    "    else:\n",
    "        rand = np.random.randint(MAX_MEM_SIZE)\n",
    "        initialStateMemory[rand] = initialState.reshape(INPUT_SIZE)\n",
    "        actionMemory[rand] = action\n",
    "        rewardMemory[rand] = reward\n",
    "        finalStateMemory[rand] = finalState.reshape(INPUT_SIZE)\n",
    "\n",
    "    memCount += 1\n",
    "        \n",
    "    return initialStateMemory,actionMemory,rewardMemory,finalStateMemory,memCount"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTargetQs(states0, actions, rewards, states1, evalnet, targetnet):\n",
    "    DISCOUNT = 0.99\n",
    "    sample_size = len(states0)\n",
    "    \n",
    "    evalnetQs = np.zeros((sample_size, OUTPUT_SIZE))\n",
    "    targetnetQs = np.zeros((sample_size, OUTPUT_SIZE))\n",
    "    currentStateQs = np.zeros((sample_size, OUTPUT_SIZE))\n",
    "    \n",
    "    evalnetQs = np.array([getPrediction(states1[i], evalnet) for i in range(sample_size)])\n",
    "    targetnetQs = np.array([getPrediction(states1[i], targetnet) for i in range(sample_size)])\n",
    "    currentStateQs = np.array([getPrediction(states0[i], evalnet) for i in range(sample_size)])\n",
    "    targetQs = currentStateQs\n",
    "    \n",
    "    argmaxActions = np.argmax(evalnetQs, axis = 1)\n",
    "    indices = np.arange(len(states0))\n",
    "    \n",
    "    targetQs[indices, actions] = rewards + DISCOUNT * targetnetQs[indices,argmaxActions]\n",
    "    \n",
    "    return targetQs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Game Logic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initializeEnv():\n",
    "    env = np.zeros((BOARD_LENGTH,BOARD_LENGTH))\n",
    "\n",
    "    for i in range(2):\n",
    "        addValue(env)\n",
    "            \n",
    "    return env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addValue(env):\n",
    "#     rand = np.random.rand()\n",
    "#     if rand > 0.1:\n",
    "#         value = 2\n",
    "#     else:\n",
    "#         value = 4\n",
    "    value = 2\n",
    "        \n",
    "    coordinate1 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "    coordinate2 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "        \n",
    "    if env[coordinate1,coordinate2] != 0:\n",
    "        getNewCoordinate(env,value)\n",
    "    else:\n",
    "        env[coordinate1,coordinate2] = value\n",
    "        \n",
    "    return env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getNewCoordinate(env,value):\n",
    "    \n",
    "    coordinate1 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "    coordinate2 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "    \n",
    "    while env[coordinate1,coordinate2] != 0:\n",
    "        coordinate1 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "        coordinate2 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "    \n",
    "    env[coordinate1,coordinate2] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def step(env,action,score):\n",
    "    \n",
    "    if action == 0:\n",
    "        env, score, validAction = actionUp(env,score)\n",
    "    if action == 1:\n",
    "        env, score, validAction = actionDown(env,score)\n",
    "    if action == 2:\n",
    "        env, score, validAction = actionLeft(env,score)\n",
    "    if action == 3:\n",
    "        env, score, validAction = actionRight(env,score)\n",
    "    \n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actionUp(env,score):\n",
    "    validAction = False\n",
    "    \n",
    "    for i in range(BOARD_LENGTH):\n",
    "        for j in range(BOARD_LENGTH):\n",
    "            row = i\n",
    "            while env[row][j] != 0 and row != 0:\n",
    "                if env[row - 1][j] != 0:\n",
    "                    if env[row - 1][j] == env[row][j]:\n",
    "                        env[row - 1][j] *= 2\n",
    "                        env[row][j] = 0\n",
    "                        score += env[row - 1][j]\n",
    "                        validAction = True\n",
    "                    break\n",
    "                temp = env[row][j]\n",
    "                env[row-1][j] = temp\n",
    "                env[row][j] = 0\n",
    "                row -= 1\n",
    "                validAction = True\n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actionDown(env,score):\n",
    "    validAction = False\n",
    "    \n",
    "    for i in range(3,-1,-1):\n",
    "        for j in range(BOARD_LENGTH):\n",
    "            row = i\n",
    "            while env[row][j] != 0 and row != 3:\n",
    "                if env[row + 1][j] != 0:\n",
    "                    if env[row + 1][j] == env[row][j]:\n",
    "                        env[row + 1][j] *= 2\n",
    "                        env[row][j] = 0\n",
    "                        score += env[row + 1][j]\n",
    "                        validAction = True\n",
    "                    break\n",
    "                temp = env[row][j]\n",
    "                env[row + 1][j] = temp\n",
    "                env[row][j] = 0\n",
    "                row += 1\n",
    "                validAction = True\n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actionLeft(env,score):\n",
    "    validAction = False\n",
    "    env = env.T\n",
    "    \n",
    "    for i in range(BOARD_LENGTH):\n",
    "        for j in range(BOARD_LENGTH):\n",
    "            row = i\n",
    "            while env[row][j] != 0 and row != 0:\n",
    "                if env[row - 1][j] != 0:\n",
    "                    if env[row - 1][j] == env[row][j]:\n",
    "                        env[row - 1][j] *= 2\n",
    "                        env[row][j] = 0\n",
    "                        score += env[row - 1][j]\n",
    "                        validAction = True\n",
    "                    break\n",
    "                temp = env[row][j]\n",
    "                env[row-1][j] = temp\n",
    "                env[row][j] = 0\n",
    "                row -= 1\n",
    "                validAction = True\n",
    "    env = env.T\n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actionRight(env,score):\n",
    "    validAction = False\n",
    "    env = env.T\n",
    "    \n",
    "    for i in range(3,-1,-1):\n",
    "        for j in range(BOARD_LENGTH):\n",
    "            row = i\n",
    "            while env[row][j] != 0 and row != 3:\n",
    "                if env[row + 1][j] != 0:\n",
    "                    if env[row + 1][j] == env[row][j]:\n",
    "                        env[row + 1][j] *= 2\n",
    "                        env[row][j] = 0\n",
    "                        score += env[row + 1][j]\n",
    "                        validAction = True\n",
    "                    break\n",
    "                temp = env[row][j]\n",
    "                env[row + 1][j] = temp\n",
    "                env[row][j] = 0\n",
    "                row += 1\n",
    "                validAction = True\n",
    "    env = env.T\n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAvailableAction(env):\n",
    "    tempEnv = np.copy(env)\n",
    "    available = False\n",
    "    \n",
    "    for i in range(tempEnv.shape[0]):\n",
    "        for j in range(tempEnv.shape[0] - 1):\n",
    "            if tempEnv[i][j] == tempEnv[i][j+1]:\n",
    "                available = True\n",
    "                \n",
    "                \n",
    "    tempEnv = tempEnv.T\n",
    "    for i in range(tempEnv.shape[0]):\n",
    "        for j in range(tempEnv.shape[0] - 1):\n",
    "            if tempEnv[i][j] == tempEnv[i][j+1]:\n",
    "                available = True\n",
    "                break\n",
    "    \n",
    "    return available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(env,agent,memCount,testFlag,score = 0):\n",
    "    \n",
    "    gameOver = False\n",
    "    while gameOver == False:\n",
    "        invalidActions = []\n",
    "        validAction = False\n",
    "        while validAction == False:\n",
    "            \n",
    "            if not testFlag:\n",
    "                previousEnv = np.copy(env)\n",
    "                previousScore = np.copy(score)\n",
    "            \n",
    "            output = getPrediction(env,agent)\n",
    "            if testFlag: \n",
    "                action = getGreedyAction(output,invalidActions) \n",
    "            else: \n",
    "                action = getAction(output,invalidActions)\n",
    "            invalidActions.append(action)\n",
    "            env, score, validAction = step(env, action, score)\n",
    "        \n",
    "        if not testFlag:\n",
    "            initialStateMemory,actionMemory,rewardMemory,finalStateMemory,memCount = addMemory(previousEnv, action, score - previousScore, env, memCount, MAX_MEM_SIZE)\n",
    "\n",
    "        env = addValue(env)\n",
    "        \n",
    "        if np.sum(env == 0) == 0:\n",
    "            gameOver = not getAvailableAction(env)\n",
    "    \n",
    "    if not testFlag:\n",
    "        return score,initialStateMemory,actionMemory,rewardMemory,finalStateMemory,memCount\n",
    "    else: \n",
    "        return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main Output Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printIter():\n",
    "    print(\"\\n === === === Iteration\",iteration,\"of\",ITERATIONS,'=== === ===\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printStats(timeLog,iteration,ITERATIONS,memCount,scoreLog,greedyScoreLog,EPISODES,TESTS):\n",
    "    print(\"\\nEstimated time remaining: \", format(np.mean(timeLog) * (ITERATIONS - iteration) / 60,'.2f'),\" Minutes\\n\")\n",
    "    print(\"Memories Elapsed: \",memCount)\n",
    "    print(\"Current Epsilon: \",format(EPSILON_RATE ** iteration,'.4f'))\n",
    "    print(\"Ave Game Score for Epoch: \", format(np.mean(scoreLog[len(scoreLog) - EPISODES : len(scoreLog)-1]) ,'.2f'))\n",
    "    print(\"Ave Greedy Score for Epoch: \", format(np.mean(greedyScoreLog[len(greedyScoreLog) - TESTS : len(greedyScoreLog)-1]) ,'.2f'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printTrainInfo():\n",
    "    print(\"Maximum memory size:\",MAX_MEM_SIZE)\n",
    "    print(\"Sample size for training:\",SAMPLE_SIZE)\n",
    "    print(\"Batch size from sample:\",BATCH_SIZE)\n",
    "    print(\"Epslon Rate:\",format(EPSILON_RATE,'.4f'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import time\n",
    "import sys\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras import layers\n",
    "from keras.layers import Dense, Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "BOARD_LENGTH = 4\n",
    "INPUT_SIZE = 16\n",
    "OUTPUT_SIZE = BOARD_LENGTH \n",
    "\n",
    "memCount = 0\n",
    "MAX_MEM_SIZE = 100_000\n",
    "SAMPLE_SIZE = MAX_MEM_SIZE // 200\n",
    "BATCH_SIZE = SAMPLE_SIZE // 100\n",
    "initialStateMemory = np.zeros((MAX_MEM_SIZE,INPUT_SIZE))\n",
    "actionMemory = np.zeros(MAX_MEM_SIZE, dtype = np.int)\n",
    "rewardMemory = np.zeros(MAX_MEM_SIZE)\n",
    "finalStateMemory = np.zeros((MAX_MEM_SIZE,INPUT_SIZE))\n",
    "\n",
    "\n",
    "gameScoreLog = []\n",
    "greedyScoreLog = []\n",
    "agent = intitializeAgent()\n",
    "target = intitializeAgent()\n",
    "updateTargetNetwork()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "ITERATIONS = 100\n",
    "EPISODES = 100\n",
    "TESTS = EPISODES // 5\n",
    "EPOCHS = 4\n",
    "ITER_PER_UPDATE = 25\n",
    "MIN_EPSILON = 0.05\n",
    "EPSILON_RATE = MIN_EPSILON ** (1./ITERATIONS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum memory size: 100000\n",
      "Sample size for training: 500\n",
      "Batch size from sample: 5\n",
      "Epslon Rate: 0.9705\n",
      "\n",
      " === === === Iteration 0 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  nan  Minutes\n",
      "\n",
      "Memories Elapsed:  13913\n",
      "Current Epsilon:  1.0000\n",
      "Ave Game Score for Epoch:  nan\n",
      "Ave Greedy Score for Epoch:  nan\n",
      "\n",
      " === === === Iteration 1 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  7.01  Minutes\n",
      "\n",
      "Memories Elapsed:  17382\n",
      "Current Epsilon:  0.9705\n",
      "Ave Game Score for Epoch:  1290.33\n",
      "Ave Greedy Score for Epoch:  795.00\n",
      "\n",
      " === === === Iteration 2 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  6.50  Minutes\n",
      "\n",
      "Memories Elapsed:  20982\n",
      "Current Epsilon:  0.9418\n",
      "Ave Game Score for Epoch:  1332.50\n",
      "Ave Greedy Score for Epoch:  753.00\n",
      "\n",
      " === === === Iteration 3 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  6.36  Minutes\n",
      "\n",
      "Memories Elapsed:  24496\n",
      "Current Epsilon:  0.9140\n",
      "Ave Game Score for Epoch:  1258.83\n",
      "Ave Greedy Score for Epoch:  1065.00\n",
      "\n",
      " === === === Iteration 4 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  6.23  Minutes\n",
      "\n",
      "Memories Elapsed:  28008\n",
      "Current Epsilon:  0.8871\n",
      "Ave Game Score for Epoch:  1299.17\n",
      "Ave Greedy Score for Epoch:  767.00\n",
      "\n",
      " === === === Iteration 5 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  6.12  Minutes\n",
      "\n",
      "Memories Elapsed:  31406\n",
      "Current Epsilon:  0.8609\n",
      "Ave Game Score for Epoch:  1235.17\n",
      "Ave Greedy Score for Epoch:  863.00\n",
      "\n",
      " === === === Iteration 6 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  6.06  Minutes\n",
      "\n",
      "Memories Elapsed:  34997\n",
      "Current Epsilon:  0.8355\n",
      "Ave Game Score for Epoch:  1325.00\n",
      "Ave Greedy Score for Epoch:  981.00\n",
      "\n",
      " === === === Iteration 7 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  6.03  Minutes\n",
      "\n",
      "Memories Elapsed:  38825\n",
      "Current Epsilon:  0.8108\n",
      "Ave Game Score for Epoch:  1458.33\n",
      "Ave Greedy Score for Epoch:  1165.00\n",
      "\n",
      " === === === Iteration 8 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.91  Minutes\n",
      "\n",
      "Memories Elapsed:  42110\n",
      "Current Epsilon:  0.7869\n",
      "Ave Game Score for Epoch:  1156.50\n",
      "Ave Greedy Score for Epoch:  1077.00\n",
      "\n",
      " === === === Iteration 9 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.81  Minutes\n",
      "\n",
      "Memories Elapsed:  45541\n",
      "Current Epsilon:  0.7637\n",
      "Ave Game Score for Epoch:  1221.50\n",
      "Ave Greedy Score for Epoch:  1181.00\n",
      "\n",
      " === === === Iteration 10 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.74  Minutes\n",
      "\n",
      "Memories Elapsed:  49041\n",
      "Current Epsilon:  0.7411\n",
      "Ave Game Score for Epoch:  1254.00\n",
      "Ave Greedy Score for Epoch:  1326.00\n",
      "\n",
      " === === === Iteration 11 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.70  Minutes\n",
      "\n",
      "Memories Elapsed:  52768\n",
      "Current Epsilon:  0.7193\n",
      "Ave Game Score for Epoch:  1440.50\n",
      "Ave Greedy Score for Epoch:  982.00\n",
      "\n",
      " === === === Iteration 12 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.61  Minutes\n",
      "\n",
      "Memories Elapsed:  56102\n",
      "Current Epsilon:  0.6980\n",
      "Ave Game Score for Epoch:  1184.50\n",
      "Ave Greedy Score for Epoch:  1271.00\n",
      "\n",
      " === === === Iteration 13 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.58  Minutes\n",
      "\n",
      "Memories Elapsed:  59755\n",
      "Current Epsilon:  0.6774\n",
      "Ave Game Score for Epoch:  1356.00\n",
      "Ave Greedy Score for Epoch:  2042.00\n",
      "\n",
      " === === === Iteration 14 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.52  Minutes\n",
      "\n",
      "Memories Elapsed:  63248\n",
      "Current Epsilon:  0.6574\n",
      "Ave Game Score for Epoch:  1272.00\n",
      "Ave Greedy Score for Epoch:  1258.00\n",
      "\n",
      " === === === Iteration 15 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.48  Minutes\n",
      "\n",
      "Memories Elapsed:  66742\n",
      "Current Epsilon:  0.6380\n",
      "Ave Game Score for Epoch:  1323.33\n",
      "Ave Greedy Score for Epoch:  2000.00\n",
      "\n",
      " === === === Iteration 16 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.41  Minutes\n",
      "\n",
      "Memories Elapsed:  70121\n",
      "Current Epsilon:  0.6192\n",
      "Ave Game Score for Epoch:  1250.00\n",
      "Ave Greedy Score for Epoch:  1526.00\n",
      "\n",
      " === === === Iteration 17 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.35  Minutes\n",
      "\n",
      "Memories Elapsed:  73684\n",
      "Current Epsilon:  0.6009\n",
      "Ave Game Score for Epoch:  1317.83\n",
      "Ave Greedy Score for Epoch:  1451.00\n",
      "\n",
      " === === === Iteration 18 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.27  Minutes\n",
      "\n",
      "Memories Elapsed:  77100\n",
      "Current Epsilon:  0.5832\n",
      "Ave Game Score for Epoch:  1181.00\n",
      "Ave Greedy Score for Epoch:  1226.00\n",
      "\n",
      " === === === Iteration 19 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.21  Minutes\n",
      "\n",
      "Memories Elapsed:  80778\n",
      "Current Epsilon:  0.5660\n",
      "Ave Game Score for Epoch:  1344.00\n",
      "Ave Greedy Score for Epoch:  1054.00\n",
      "\n",
      " === === === Iteration 20 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.15  Minutes\n",
      "\n",
      "Memories Elapsed:  84138\n",
      "Current Epsilon:  0.5493\n",
      "Ave Game Score for Epoch:  1188.33\n",
      "Ave Greedy Score for Epoch:  1235.00\n",
      "\n",
      " === === === Iteration 21 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.09  Minutes\n",
      "\n",
      "Memories Elapsed:  87693\n",
      "Current Epsilon:  0.5331\n",
      "Ave Game Score for Epoch:  1263.50\n",
      "Ave Greedy Score for Epoch:  1492.00\n",
      "\n",
      " === === === Iteration 22 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.03  Minutes\n",
      "\n",
      "Memories Elapsed:  91221\n",
      "Current Epsilon:  0.5173\n",
      "Ave Game Score for Epoch:  1275.67\n",
      "Ave Greedy Score for Epoch:  1450.00\n",
      "\n",
      " === === === Iteration 23 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.99  Minutes\n",
      "\n",
      "Memories Elapsed:  94926\n",
      "Current Epsilon:  0.5021\n",
      "Ave Game Score for Epoch:  1432.17\n",
      "Ave Greedy Score for Epoch:  1668.00\n",
      "\n",
      " === === === Iteration 24 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.94  Minutes\n",
      "\n",
      "Memories Elapsed:  98452\n",
      "Current Epsilon:  0.4873\n",
      "Ave Game Score for Epoch:  1290.83\n",
      "Ave Greedy Score for Epoch:  1936.00\n",
      "\n",
      " === === === Iteration 25 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.89  Minutes\n",
      "\n",
      "Memories Elapsed:  101887\n",
      "Current Epsilon:  0.4729\n",
      "Ave Game Score for Epoch:  1248.50\n",
      "Ave Greedy Score for Epoch:  3174.00\n",
      "\n",
      " === === === Iteration 26 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.83  Minutes\n",
      "\n",
      "Memories Elapsed:  105410\n",
      "Current Epsilon:  0.4589\n",
      "Ave Game Score for Epoch:  1311.00\n",
      "Ave Greedy Score for Epoch:  1205.00\n",
      "\n",
      " === === === Iteration 27 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.77  Minutes\n",
      "\n",
      "Memories Elapsed:  108683\n",
      "Current Epsilon:  0.4454\n",
      "Ave Game Score for Epoch:  1139.00\n",
      "Ave Greedy Score for Epoch:  2084.00\n",
      "\n",
      " === === === Iteration 28 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.71  Minutes\n",
      "\n",
      "Memories Elapsed:  112064\n",
      "Current Epsilon:  0.4322\n",
      "Ave Game Score for Epoch:  1162.50\n",
      "Ave Greedy Score for Epoch:  1746.00\n",
      "\n",
      " === === === Iteration 29 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.65  Minutes\n",
      "\n",
      "Memories Elapsed:  115441\n",
      "Current Epsilon:  0.4195\n",
      "Ave Game Score for Epoch:  1212.17\n",
      "Ave Greedy Score for Epoch:  1886.00\n",
      "\n",
      " === === === Iteration 30 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.59  Minutes\n",
      "\n",
      "Memories Elapsed:  118745\n",
      "Current Epsilon:  0.4071\n",
      "Ave Game Score for Epoch:  1151.17\n",
      "Ave Greedy Score for Epoch:  2691.00\n",
      "\n",
      " === === === Iteration 31 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.53  Minutes\n",
      "\n",
      "Memories Elapsed:  122064\n",
      "Current Epsilon:  0.3951\n",
      "Ave Game Score for Epoch:  1168.33\n",
      "Ave Greedy Score for Epoch:  2252.00\n",
      "\n",
      " === === === Iteration 32 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.50  Minutes\n",
      "\n",
      "Memories Elapsed:  125789\n",
      "Current Epsilon:  0.3834\n",
      "Ave Game Score for Epoch:  1351.67\n",
      "Ave Greedy Score for Epoch:  3781.00\n",
      "\n",
      " === === === Iteration 33 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.44  Minutes\n",
      "\n",
      "Memories Elapsed:  129425\n",
      "Current Epsilon:  0.3721\n",
      "Ave Game Score for Epoch:  1305.33\n",
      "Ave Greedy Score for Epoch:  1483.00\n",
      "\n",
      " === === === Iteration 34 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.40  Minutes\n",
      "\n",
      "Memories Elapsed:  133281\n",
      "Current Epsilon:  0.3611\n",
      "Ave Game Score for Epoch:  1366.50\n",
      "Ave Greedy Score for Epoch:  3172.00\n",
      "\n",
      " === === === Iteration 35 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.35  Minutes\n",
      "\n",
      "Memories Elapsed:  137018\n",
      "Current Epsilon:  0.3505\n",
      "Ave Game Score for Epoch:  1380.83\n",
      "Ave Greedy Score for Epoch:  2395.00\n",
      "\n",
      " === === === Iteration 36 of 100 === === ===\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.30  Minutes\n",
      "\n",
      "Memories Elapsed:  140485\n",
      "Current Epsilon:  0.3401\n",
      "Ave Game Score for Epoch:  1286.17\n",
      "Ave Greedy Score for Epoch:  2159.00\n",
      "\n",
      " === === === Iteration 37 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.24  Minutes\n",
      "\n",
      "Memories Elapsed:  144206\n",
      "Current Epsilon:  0.3301\n",
      "Ave Game Score for Epoch:  1401.17\n",
      "Ave Greedy Score for Epoch:  1766.00\n",
      "\n",
      " === === === Iteration 38 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.19  Minutes\n",
      "\n",
      "Memories Elapsed:  147636\n",
      "Current Epsilon:  0.3203\n",
      "Ave Game Score for Epoch:  1190.17\n",
      "Ave Greedy Score for Epoch:  2124.00\n",
      "\n",
      " === === === Iteration 39 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.14  Minutes\n",
      "\n",
      "Memories Elapsed:  151533\n",
      "Current Epsilon:  0.3109\n",
      "Ave Game Score for Epoch:  1443.33\n",
      "Ave Greedy Score for Epoch:  2628.00\n",
      "\n",
      " === === === Iteration 40 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.09  Minutes\n",
      "\n",
      "Memories Elapsed:  155238\n",
      "Current Epsilon:  0.3017\n",
      "Ave Game Score for Epoch:  1371.67\n",
      "Ave Greedy Score for Epoch:  3259.00\n",
      "\n",
      " === === === Iteration 41 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.03  Minutes\n",
      "\n",
      "Memories Elapsed:  159302\n",
      "Current Epsilon:  0.2928\n",
      "Ave Game Score for Epoch:  1525.67\n",
      "Ave Greedy Score for Epoch:  1586.00\n",
      "\n",
      " === === === Iteration 42 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.96  Minutes\n",
      "\n",
      "Memories Elapsed:  162609\n",
      "Current Epsilon:  0.2842\n",
      "Ave Game Score for Epoch:  1151.00\n",
      "Ave Greedy Score for Epoch:  1828.00\n",
      "\n",
      " === === === Iteration 43 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.89  Minutes\n",
      "\n",
      "Memories Elapsed:  166227\n",
      "Current Epsilon:  0.2758\n",
      "Ave Game Score for Epoch:  1247.33\n",
      "Ave Greedy Score for Epoch:  2555.00\n",
      "\n",
      " === === === Iteration 44 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.83  Minutes\n",
      "\n",
      "Memories Elapsed:  170221\n",
      "Current Epsilon:  0.2676\n",
      "Ave Game Score for Epoch:  1504.33\n",
      "Ave Greedy Score for Epoch:  2163.00\n",
      "\n",
      " === === === Iteration 45 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.76  Minutes\n",
      "\n",
      "Memories Elapsed:  174067\n",
      "Current Epsilon:  0.2597\n",
      "Ave Game Score for Epoch:  1459.67\n",
      "Ave Greedy Score for Epoch:  2762.00\n",
      "\n",
      " === === === Iteration 46 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.70  Minutes\n",
      "\n",
      "Memories Elapsed:  177778\n",
      "Current Epsilon:  0.2521\n",
      "Ave Game Score for Epoch:  1302.00\n",
      "Ave Greedy Score for Epoch:  2804.00\n",
      "\n",
      " === === === Iteration 47 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.63  Minutes\n",
      "\n",
      "Memories Elapsed:  181344\n",
      "Current Epsilon:  0.2446\n",
      "Ave Game Score for Epoch:  1281.50\n",
      "Ave Greedy Score for Epoch:  2454.00\n",
      "\n",
      " === === === Iteration 48 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.57  Minutes\n",
      "\n",
      "Memories Elapsed:  185303\n",
      "Current Epsilon:  0.2374\n",
      "Ave Game Score for Epoch:  1532.00\n",
      "Ave Greedy Score for Epoch:  1562.00\n",
      "\n",
      " === === === Iteration 49 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.51  Minutes\n",
      "\n",
      "Memories Elapsed:  189041\n",
      "Current Epsilon:  0.2304\n",
      "Ave Game Score for Epoch:  1330.50\n",
      "Ave Greedy Score for Epoch:  2486.00\n",
      "\n",
      " === === === Iteration 50 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.45  Minutes\n",
      "\n",
      "Memories Elapsed:  192799\n",
      "Current Epsilon:  0.2236\n",
      "Ave Game Score for Epoch:  1366.67\n",
      "Ave Greedy Score for Epoch:  2818.00\n",
      "\n",
      " === === === Iteration 51 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.38  Minutes\n",
      "\n",
      "Memories Elapsed:  196821\n",
      "Current Epsilon:  0.2170\n",
      "Ave Game Score for Epoch:  1509.50\n",
      "Ave Greedy Score for Epoch:  1673.00\n",
      "\n",
      " === === === Iteration 52 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.32  Minutes\n",
      "\n",
      "Memories Elapsed:  200969\n",
      "Current Epsilon:  0.2106\n",
      "Ave Game Score for Epoch:  1603.50\n",
      "Ave Greedy Score for Epoch:  2366.00\n",
      "\n",
      " === === === Iteration 53 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.26  Minutes\n",
      "\n",
      "Memories Elapsed:  205070\n",
      "Current Epsilon:  0.2044\n",
      "Ave Game Score for Epoch:  1586.83\n",
      "Ave Greedy Score for Epoch:  3414.00\n",
      "\n",
      " === === === Iteration 54 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.20  Minutes\n",
      "\n",
      "Memories Elapsed:  209085\n",
      "Current Epsilon:  0.1984\n",
      "Ave Game Score for Epoch:  1477.00\n",
      "Ave Greedy Score for Epoch:  3115.00\n",
      "\n",
      " === === === Iteration 55 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.14  Minutes\n",
      "\n",
      "Memories Elapsed:  213405\n",
      "Current Epsilon:  0.1925\n",
      "Ave Game Score for Epoch:  1712.83\n",
      "Ave Greedy Score for Epoch:  3024.00\n",
      "\n",
      " === === === Iteration 56 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.07  Minutes\n",
      "\n",
      "Memories Elapsed:  217052\n",
      "Current Epsilon:  0.1868\n",
      "Ave Game Score for Epoch:  1309.83\n",
      "Ave Greedy Score for Epoch:  2258.00\n",
      "\n",
      " === === === Iteration 57 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.00  Minutes\n",
      "\n",
      "Memories Elapsed:  221120\n",
      "Current Epsilon:  0.1813\n",
      "Ave Game Score for Epoch:  1475.00\n",
      "Ave Greedy Score for Epoch:  2990.00\n",
      "\n",
      " === === === Iteration 58 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.94  Minutes\n",
      "\n",
      "Memories Elapsed:  224955\n",
      "Current Epsilon:  0.1760\n",
      "Ave Game Score for Epoch:  1413.17\n",
      "Ave Greedy Score for Epoch:  3364.00\n",
      "\n",
      " === === === Iteration 59 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.87  Minutes\n",
      "\n",
      "Memories Elapsed:  229104\n",
      "Current Epsilon:  0.1708\n",
      "Ave Game Score for Epoch:  1629.83\n",
      "Ave Greedy Score for Epoch:  2852.00\n",
      "\n",
      " === === === Iteration 60 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.82  Minutes\n",
      "\n",
      "Memories Elapsed:  232795\n",
      "Current Epsilon:  0.1657\n",
      "Ave Game Score for Epoch:  1327.17\n",
      "Ave Greedy Score for Epoch:  3123.00\n",
      "\n",
      " === === === Iteration 61 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.75  Minutes\n",
      "\n",
      "Memories Elapsed:  236275\n",
      "Current Epsilon:  0.1608\n",
      "Ave Game Score for Epoch:  1171.33\n",
      "Ave Greedy Score for Epoch:  2893.00\n",
      "\n",
      " === === === Iteration 62 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.68  Minutes\n",
      "\n",
      "Memories Elapsed:  240499\n",
      "Current Epsilon:  0.1561\n",
      "Ave Game Score for Epoch:  1568.67\n",
      "Ave Greedy Score for Epoch:  3185.00\n",
      "\n",
      " === === === Iteration 63 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.62  Minutes\n",
      "\n",
      "Memories Elapsed:  245161\n",
      "Current Epsilon:  0.1515\n",
      "Ave Game Score for Epoch:  1890.83\n",
      "Ave Greedy Score for Epoch:  2233.00\n",
      "\n",
      " === === === Iteration 64 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.55  Minutes\n",
      "\n",
      "Memories Elapsed:  249602\n",
      "Current Epsilon:  0.1470\n",
      "Ave Game Score for Epoch:  1757.83\n",
      "Ave Greedy Score for Epoch:  2448.00\n",
      "\n",
      " === === === Iteration 65 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.48  Minutes\n",
      "\n",
      "Memories Elapsed:  253605\n",
      "Current Epsilon:  0.1427\n",
      "Ave Game Score for Epoch:  1457.83\n",
      "Ave Greedy Score for Epoch:  1989.00\n",
      "\n",
      " === === === Iteration 66 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.42  Minutes\n",
      "\n",
      "Memories Elapsed:  257905\n",
      "Current Epsilon:  0.1385\n",
      "Ave Game Score for Epoch:  1634.50\n",
      "Ave Greedy Score for Epoch:  2091.00\n",
      "\n",
      " === === === Iteration 67 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.35  Minutes\n",
      "\n",
      "Memories Elapsed:  261930\n",
      "Current Epsilon:  0.1344\n",
      "Ave Game Score for Epoch:  1456.00\n",
      "Ave Greedy Score for Epoch:  2079.00\n",
      "\n",
      " === === === Iteration 68 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.28  Minutes\n",
      "\n",
      "Memories Elapsed:  266322\n",
      "Current Epsilon:  0.1304\n",
      "Ave Game Score for Epoch:  1675.83\n",
      "Ave Greedy Score for Epoch:  2716.00\n",
      "\n",
      " === === === Iteration 69 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.22  Minutes\n",
      "\n",
      "Memories Elapsed:  271147\n",
      "Current Epsilon:  0.1266\n",
      "Ave Game Score for Epoch:  1985.50\n",
      "Ave Greedy Score for Epoch:  2239.00\n",
      "\n",
      " === === === Iteration 70 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.15  Minutes\n",
      "\n",
      "Memories Elapsed:  275622\n",
      "Current Epsilon:  0.1228\n",
      "Ave Game Score for Epoch:  1732.67\n",
      "Ave Greedy Score for Epoch:  1960.00\n",
      "\n",
      " === === === Iteration 71 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.08  Minutes\n",
      "\n",
      "Memories Elapsed:  280500\n",
      "Current Epsilon:  0.1192\n",
      "Ave Game Score for Epoch:  2008.67\n",
      "Ave Greedy Score for Epoch:  2665.00\n",
      "\n",
      " === === === Iteration 72 of 100 === === ===\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.01  Minutes\n",
      "\n",
      "Memories Elapsed:  284892\n",
      "Current Epsilon:  0.1157\n",
      "Ave Game Score for Epoch:  1663.33\n",
      "Ave Greedy Score for Epoch:  3274.00\n",
      "\n",
      " === === === Iteration 73 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.95  Minutes\n",
      "\n",
      "Memories Elapsed:  289215\n",
      "Current Epsilon:  0.1123\n",
      "Ave Game Score for Epoch:  1591.83\n",
      "Ave Greedy Score for Epoch:  3251.00\n",
      "\n",
      " === === === Iteration 74 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.88  Minutes\n",
      "\n",
      "Memories Elapsed:  293568\n",
      "Current Epsilon:  0.1090\n",
      "Ave Game Score for Epoch:  1632.33\n",
      "Ave Greedy Score for Epoch:  2439.00\n",
      "\n",
      " === === === Iteration 75 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.81  Minutes\n",
      "\n",
      "Memories Elapsed:  298072\n",
      "Current Epsilon:  0.1057\n",
      "Ave Game Score for Epoch:  1733.00\n",
      "Ave Greedy Score for Epoch:  1766.00\n",
      "\n",
      " === === === Iteration 76 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.74  Minutes\n",
      "\n",
      "Memories Elapsed:  302550\n",
      "Current Epsilon:  0.1026\n",
      "Ave Game Score for Epoch:  1732.17\n",
      "Ave Greedy Score for Epoch:  2834.00\n",
      "\n",
      " === === === Iteration 77 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.67  Minutes\n",
      "\n",
      "Memories Elapsed:  307116\n",
      "Current Epsilon:  0.0996\n",
      "Ave Game Score for Epoch:  1746.83\n",
      "Ave Greedy Score for Epoch:  2659.00\n",
      "\n",
      " === === === Iteration 78 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.61  Minutes\n",
      "\n",
      "Memories Elapsed:  311910\n",
      "Current Epsilon:  0.0966\n",
      "Ave Game Score for Epoch:  1952.00\n",
      "Ave Greedy Score for Epoch:  4725.00\n",
      "\n",
      " === === === Iteration 79 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.54  Minutes\n",
      "\n",
      "Memories Elapsed:  316612\n",
      "Current Epsilon:  0.0938\n",
      "Ave Game Score for Epoch:  1914.83\n",
      "Ave Greedy Score for Epoch:  2074.00\n",
      "\n",
      " === === === Iteration 80 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.47  Minutes\n",
      "\n",
      "Memories Elapsed:  321942\n",
      "Current Epsilon:  0.0910\n",
      "Ave Game Score for Epoch:  2266.00\n",
      "Ave Greedy Score for Epoch:  1835.00\n",
      "\n",
      " === === === Iteration 81 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.40  Minutes\n",
      "\n",
      "Memories Elapsed:  327108\n",
      "Current Epsilon:  0.0883\n",
      "Ave Game Score for Epoch:  2132.17\n",
      "Ave Greedy Score for Epoch:  2133.00\n",
      "\n",
      " === === === Iteration 82 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.33  Minutes\n",
      "\n",
      "Memories Elapsed:  332153\n",
      "Current Epsilon:  0.0857\n",
      "Ave Game Score for Epoch:  2016.33\n",
      "Ave Greedy Score for Epoch:  1938.00\n",
      "\n",
      " === === === Iteration 83 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.26  Minutes\n",
      "\n",
      "Memories Elapsed:  337221\n",
      "Current Epsilon:  0.0832\n",
      "Ave Game Score for Epoch:  2110.83\n",
      "Ave Greedy Score for Epoch:  3461.00\n",
      "\n",
      " === === === Iteration 84 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.19  Minutes\n",
      "\n",
      "Memories Elapsed:  342103\n",
      "Current Epsilon:  0.0807\n",
      "Ave Game Score for Epoch:  1970.50\n",
      "Ave Greedy Score for Epoch:  2774.00\n",
      "\n",
      " === === === Iteration 85 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.12  Minutes\n",
      "\n",
      "Memories Elapsed:  346473\n",
      "Current Epsilon:  0.0784\n",
      "Ave Game Score for Epoch:  1660.67\n",
      "Ave Greedy Score for Epoch:  2693.00\n",
      "\n",
      " === === === Iteration 86 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.04  Minutes\n",
      "\n",
      "Memories Elapsed:  350697\n",
      "Current Epsilon:  0.0761\n",
      "Ave Game Score for Epoch:  1558.33\n",
      "Ave Greedy Score for Epoch:  2829.00\n",
      "\n",
      " === === === Iteration 87 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.97  Minutes\n",
      "\n",
      "Memories Elapsed:  355621\n",
      "Current Epsilon:  0.0738\n",
      "Ave Game Score for Epoch:  1939.00\n",
      "Ave Greedy Score for Epoch:  3512.00\n",
      "\n",
      " === === === Iteration 88 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.90  Minutes\n",
      "\n",
      "Memories Elapsed:  360565\n",
      "Current Epsilon:  0.0716\n",
      "Ave Game Score for Epoch:  1981.33\n",
      "Ave Greedy Score for Epoch:  2642.00\n",
      "\n",
      " === === === Iteration 89 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.83  Minutes\n",
      "\n",
      "Memories Elapsed:  366435\n",
      "Current Epsilon:  0.0695\n",
      "Ave Game Score for Epoch:  2583.83\n",
      "Ave Greedy Score for Epoch:  3316.00\n",
      "\n",
      " === === === Iteration 90 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.75  Minutes\n",
      "\n",
      "Memories Elapsed:  371490\n",
      "Current Epsilon:  0.0675\n",
      "Ave Game Score for Epoch:  2138.33\n",
      "Ave Greedy Score for Epoch:  3338.00\n",
      "\n",
      " === === === Iteration 91 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.68  Minutes\n",
      "\n",
      "Memories Elapsed:  376035\n",
      "Current Epsilon:  0.0655\n",
      "Ave Game Score for Epoch:  1807.00\n",
      "Ave Greedy Score for Epoch:  3359.00\n",
      "\n",
      " === === === Iteration 92 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.60  Minutes\n",
      "\n",
      "Memories Elapsed:  380658\n",
      "Current Epsilon:  0.0635\n",
      "Ave Game Score for Epoch:  1754.33\n",
      "Ave Greedy Score for Epoch:  2579.00\n",
      "\n",
      " === === === Iteration 93 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.53  Minutes\n",
      "\n",
      "Memories Elapsed:  385456\n",
      "Current Epsilon:  0.0617\n",
      "Ave Game Score for Epoch:  1858.00\n",
      "Ave Greedy Score for Epoch:  2601.00\n",
      "\n",
      " === === === Iteration 94 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.46  Minutes\n",
      "\n",
      "Memories Elapsed:  390223\n",
      "Current Epsilon:  0.0598\n",
      "Ave Game Score for Epoch:  1867.50\n",
      "Ave Greedy Score for Epoch:  3235.00\n",
      "\n",
      " === === === Iteration 95 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.38  Minutes\n",
      "\n",
      "Memories Elapsed:  395074\n",
      "Current Epsilon:  0.0581\n",
      "Ave Game Score for Epoch:  1945.50\n",
      "Ave Greedy Score for Epoch:  2748.00\n",
      "\n",
      " === === === Iteration 96 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.30  Minutes\n",
      "\n",
      "Memories Elapsed:  400066\n",
      "Current Epsilon:  0.0564\n",
      "Ave Game Score for Epoch:  2022.67\n",
      "Ave Greedy Score for Epoch:  2412.00\n",
      "\n",
      " === === === Iteration 97 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.23  Minutes\n",
      "\n",
      "Memories Elapsed:  405886\n",
      "Current Epsilon:  0.0547\n",
      "Ave Game Score for Epoch:  2474.17\n",
      "Ave Greedy Score for Epoch:  2606.00\n",
      "\n",
      " === === === Iteration 98 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.15  Minutes\n",
      "\n",
      "Memories Elapsed:  411407\n",
      "Current Epsilon:  0.0531\n",
      "Ave Game Score for Epoch:  2298.50\n",
      "Ave Greedy Score for Epoch:  2174.00\n",
      "\n",
      " === === === Iteration 99 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.08  Minutes\n",
      "\n",
      "Memories Elapsed:  417093\n",
      "Current Epsilon:  0.0515\n",
      "Ave Game Score for Epoch:  2402.67\n",
      "Ave Greedy Score for Epoch:  2462.00\n"
     ]
    }
   ],
   "source": [
    "startTime = time.time()\n",
    "timeLog = []\n",
    "\n",
    "printTrainInfo()\n",
    "\n",
    "for iteration in range(ITERATIONS):\n",
    "    printIter()\n",
    "    iterationStart = time.time()\n",
    "    \n",
    "    for i in range(EPISODES):\n",
    "        score,initialStateMemory,actionMemory,rewardMemory,finalStateMemory,memCount = run(initializeEnv(),agent,memCount,False)\n",
    "        gameScoreLog.append(score)\n",
    "    \n",
    "    for i in range(TESTS):\n",
    "        score = run(initializeEnv(),agent,memCount,True)\n",
    "        greedyScoreLog.append(score)\n",
    "        \n",
    "    train(initialStateMemory,actionMemory,rewardMemory,finalStateMemory,agent,target,EPOCHS)\n",
    "    timeLog.append(time.time() - iterationStart)\n",
    "    \n",
    "    if iteration % ITER_PER_UPDATE == 0:\n",
    "        updateTargetNetwork()\n",
    "    \n",
    "    if EPISODES != 25:\n",
    "        startTime = time.time()\n",
    "        timeLog = []\n",
    "        greedyScoreLog = []\n",
    "        gameScoreLog = []\n",
    "    \n",
    "    printStats(timeLog,iteration,ITERATIONS,memCount,gameScoreLog,greedyScoreLog,EPISODES,TESTS)\n",
    "    \n",
    "    EPISODES = 25\n",
    "    TESTS = EPISODES // 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmAAAAFpCAYAAAA7jJSFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X20XHV97/HPN8kBDtqbgxKFPGBSm2IxRNADem9c99qkPPiAUGuP1FZpS0vvrUpl9aKJtwtSWkuE1gBtda2IVGhVTJViFL0pJbReWVflxGAAkQsFap4qQUisJpIHvvePvQ85Z87eM3vPfp79fq111jnzO3tm9szsmf2d3+/7+/7M3QUAAIDyzKh6BwAAANqGAAwAAKBkBGAAAAAlIwADAAAoGQEYAABAyQjAAAAASkYABgAAUDICMAAAgJIRgAEAAJSMAAwAAKBks6regW6OP/54X7hwYdW7AQAA0NPmzZufcvc5SbatdQC2cOFCjY+PV70bAAAAPZnZvyXdliFIAACAkhGAAQAAlIwADAAAoGQEYAAAACUjAAMAACgZARgAAEDJCMAAAABKRgAGAABQMgIwAACAkhGAAQAAlIwADAAAoGQEYAAA9GPremntEmn1SPB76/qq9wgNUuvFuAEAqKWt66UvXSod3B9c3rstuCxJS8eq2y80Bj1gAACkdddVR4KvCQf3B+1AAgRgAACktXd7unagAwEYAABpzZ6frh3oQAAGAEBaK66Qhoantg0NB+1AAgRgAACktXRMOu8GafYCSRb8Pu8GEvCRGLMgAQDox9IxAi70jR4wAACAkhGAAQAAlIwADAAAoGQEYAAAACUjAAMAACgZARgAAEDJCMAAAABKRgAGAABQMgIwAACAkhGAAQAAlIwADAAAoGQEYAAAACUjAAMAAChZ4gDMzGaa2RYz+3J4eZGZfdPMHjGzz5nZUWH70eHlR8P/L5x0G6vC9ofN7Jy8HwwAAEATpOkB+wNJD026/BFJa919saRnJF0ctl8s6Rl3/zlJa8PtZGanSLpQ0islnSvpY2Y2M9vuAwAANE+iAMzM5kt6s6Qbw8smabmkz4eb3CzpgvDv88PLCv+/Itz+fEm3uvuz7v64pEclnZnHgwAAAGiSpD1g10n6gKTnwssvlrTH3Q+Fl7dLmhf+PU/SNkkK/7833P759ojrAAAAtEbPAMzM3iLpSXffPLk5YlPv8b9u15l8f5eY2biZje/evbvX7gEAADROkh6wZZLeamZPSLpVwdDjdZJGzGxWuM18STvDv7dLWiBJ4f9nS3p6cnvEdZ7n7uvcfdTdR+fMmZP6AQEAANRdzwDM3Ve5+3x3X6ggiX6Tu/+6pLslvT3c7CJJXwz/3hBeVvj/Te7uYfuF4SzJRZIWS/pWbo8EAACgIWb13iTWByXdamZ/KmmLpE+G7Z+U9Ldm9qiCnq8LJcndHzSz9ZK+K+mQpPe4++EM9w8AANBIFnRO1dPo6KiPj49XvRsAAAA9mdlmdx9Nsi2V8AEAAEpGAAYAAFAyAjAAAICSEYABAACUjAAMAACgZARgAAAAJSMAAwAAKBkBGAAAQMkIwAAAAEpGAAYAAFAyAjAAyMvW9dLaJdLqkeD31vVV7xGAmsqyGDcAYMLW9dKXLpUO7g8u790WXJakpWPV7deg27peuusqae92afZ8acUVPN9oBHrAACAPd111JPiacHB/0I5iTAS9e7dJ8iNBLz2PaAACMADIw97t6dqRHUEvGowADADyMHt+unZkR9CLBiMAA4A8rLhCGhqe2jY0HLSjGAS9aDACMADIw9Ix6bwbpNkLJFnw+7wbSAgvEkEvGoxZkACQl6VjBFxlmniumQWJBiIAAwA0F0EvGoohSAAAgJIRgAEAAJSMAAwAAKBkBGAAAAAlIwADAAAoGQEYAABAyQjAAAAASkYABgAAUDICMAAAgJIRgAEAgGhb10trl0irR4LfW9dXvUcDg6WIAADAdFvXS1+6VDq4P7i8d1twWWL5pxzQAwYAAKa766ojwdeEg/uDdmRGAAYAAKbbuz1dO1LpGYCZ2TFm9i0z+46ZPWhmfxy2f8rMHjez+8Kf08J2M7MbzOxRM9tqZq+edFsXmdkj4c9FxT0sACgYuTEYdLPnp2tHKklywJ6VtNzdf2xmQ5K+bmZfDf93ubt/vmP7N0paHP68VtLHJb3WzF4k6UpJo5Jc0mYz2+Duz+TxQACgNOTGoA1WXDH1OJekoeGgHZn17AHzwI/Di0Phj3e5yvmSbgmv9w1JI2Z2oqRzJN3p7k+HQdedks7NtvsAUAFyY9AGS8ek826QZi+QZMHv827gS0ZOEs2CNLOZkjZL+jlJf+3u3zSz/yHpw2Z2haS7JK1092clzZO0bdLVt4dtce0A0CzkxqAtlo4RcBUkURK+ux9299MkzZd0ppktkbRK0isknSHpRZI+GG5uUTfRpX0KM7vEzMbNbHz37t1Jdg8AykVuDICMUs2CdPc9kv5Z0rnuviscZnxW0t9IOjPcbLukBZOuNl/Szi7tnfexzt1H3X10zpw5aXYPAMqx4oogF2YycmPQFEwgqYUksyDnmNlI+PewpF+S9L0wr0tmZpIukPRAeJUNkt4dzoZ8naS97r5L0kZJZ5vZcWZ2nKSzwzYAaBZyY9BUExNI9m6T5EcmkBCElS5JDtiJkm4O88BmSFrv7l82s01mNkfB0OJ9kv57uP1XJL1J0qOS9kn6LUly96fN7E8k3Rtud5W7P53fQwGAEpEbgybqNoGE47lUPQMwd98q6fSI9uUx27uk98T87yZJN6XcRwAAkAcmkNQGlfABAKizPHO2mEBSGyzGDQB1tXV9MDS0d3twglxxBcNEbZO16G/nMbT4bOk7n6G4ag3QAwYAdUSyNKRsRX+jjqHvfEZ61TuZQFID9IABQB2RLA0pW85W3DH0yD9Klz0QfR2Uhh4wAKgjkqUhZcvZ4hiqNQIwAKgjkqUhZSv6yzFUawRgAFBHVNuHlK3oL8dQrZEDBtQBs93QaeL157hAv0V/OYZqzYK6qfU0Ojrq4+PjVe8GUKzOaeZS8C2VmUkA0ChmttndR5NsyxAkULUs08wBAI1EAAZUjZlKANA6BGBA1ZipBACtQwAGVI2ZSgDQOgRgQNWyTDMHADQSZSiAOuh3mjkAoJHoAQMAACgZARgAAEDJCMAAAABKRgAGAABQMgIwAACAkhGAAQAAlIwADADqYOt6ae0SafVI8Hvr+qr3CECBqAMGAFXbul760qVHFmXfuy24LFEfDhhQ9IABQNXuuupI8DXh4P6gHcBAIgADgKrt3Z6uHUDjEYABQNVmz0/XDqDxCMAAoGorrpCGhqe2DQ0H7SgXkyFQEpLwAaBqE4n2d10VDDvOnh8EXyTgl4vJEMlsXc+xmgNz96r3Idbo6KiPj49XvRsAgDZYuyQIujrNXiBd9kD5+1NHnUGqFPTWnncDQZgkM9vs7qNJtmUIEgAAickQSTBjNzcEYECTkJ8CFIfJEL3FBqnb+GxKiQAMaIqJrv+92yT5kfwUPuiAfDAZorfYYNT4bEqJAAxoCrr+gWItHQtymWYvkGTBb3KbpooKUmWSOvLJ+WzqqecsSDM7RtLXJB0dbv95d7/SzBZJulXSiyR9W9K73P2AmR0t6RZJr5H0Q0nvcPcnwttaJeliSYclXeruG/N/SEACTZzFQ34K8tDEY79MS8d4PrqJmrEbNXFB4rOphyQ9YM9KWu7ur5J0mqRzzex1kj4iaa27L5b0jILASuHvZ9z95yStDbeTmZ0i6UJJr5R0rqSPmdnMPB8MkEhTh/LIT0FWTT32US9Lx4JZoav3BL9nL4jejs+mrnoGYB74cXhxKPxxScslfT5sv1nSBeHf54eXFf5/hZlZ2H6ruz/r7o9LelTSmbk8CiCNpg7lkZ+CrJp67KPe+GzqS6IcMDObaWb3SXpS0p2S/lXSHnc/FG6yXdK88O95krZJUvj/vZJePLk94jpAeZo6lEd+CrJq6rGPeuOzqS+JKuG7+2FJp5nZiKR/kPQLUZuFvy3mf3HtU5jZJZIukaSTTjopye4B6cTlLDShu5z8FGTR5GMf9cZnU2qpZkG6+x5J/yzpdZJGzGwigJsvaWf493ZJCyQp/P9sSU9Pbo+4zuT7WOfuo+4+OmfOnDS7ByRDdznaimMfqI2eAZiZzQl7vmRmw5J+SdJDku6W9PZws4skfTH8e0N4WeH/N3mw3tEGSRea2dHhDMrFkr6V1wMBEqO7vHkoQNufzudN4thH/bT0/d1zLUgzW6ogqX6mgoBtvbtfZWY/qyNlKLZI+g13fzYsW/G3kk5X0PN1obs/Ft7W/5L025IOSXq/u3+1232zFiQA1p7rE88bmmDAjtM0a0GyGDeAemOB5P7wvKEJBuw4ZTFuAIODmXv94XlDE7T4OCUAA1BvFKDtD88bmqDFxykBGIB6Y+Zef3jeULUkyfUtPk4T1QEDgMpErT3H+oW98byhSp3J9RPLXklTj8EWH6ck4QNIhkWc+8PzhjYasOT6pNIk4dMDBqC3pN9mMRXPG9qqxcn1SZEDBqA3FnHuD88b2qrFyfVJEYAB6I1vs/3heUNR6l49vsXJ9UkRgAHojW+z/eF5QxEmhrb3bpPkR4a26xSEseRbTwRgAHrj22x/ynze6t4jgvw0ZWh76ViQcL96T/Cb4GsKAjAAvfFttj9lPW9N6BFBftoytD3gXyooQwEATdfSKf+xBr30Rxte74Yu0s1akADQJm3pEUmiDb2BbUgJaMowawYEYADQdCT7H9GCE3crUgJa8KWCQqwAyjHow0JVWnFF9HDNIPWIJNWCE7ek4L0zyO+f2fNjhlkH50sFPWAAiteGYaEqtaFHJCl6AwdDC4ZZ6QEDULxuw0JtDBKKMOg9IknRGzgYWrBINwEYgOK1ZVgI1WvBibs1or5UDFAqAwEYgOIVlc8xQB/GyBG9gYNpwBa3JwcMQPGKyOcgrwyYasALlw7aDFcCMKBsg/4hGaWIJPE0H8ZtfM7RLm34QjJgqQwMQQJlGrAu9FTyHhZK+mHc5uccRwz6cHUbJroMWGkKesCAMjWlC70JPUZJyw005TlHcegdGgwDVpqCAAwoUxM+JJtyskr6YdyE5xzFakMQ3ob6ZwNW744hSKBMTehCr3ooI+lQUdJyA014zlGsNgThbal/NkAzXAnAgDI14UOyypNV2nytJB/Gcc/54rOD4dUm5gQNej5T3toQhFP/rHEIwIAyNeFDssqTVRG9b1HP+eKzpe98ppmJ+UwqSK8JX3zyUEDv0O1bdujajQ9r5579mjsyrMvPOVkXnD6v/xvky8PzzN2r3odYo6OjPj4+XvVuAO3SeYKXgpNVGbkWq0ckRX0mmbR6T373s3ZJTJC5QLrsgfzupwhN3vcqceJP7fYtO7Tqtvu1/+Dh59uGh2bq6red2l8QVuVnS0nMbLO7jybZlh4wAFNV2UtXVu9bk3OCmrzvVRqg3KGyXLvx4SnBlyTtP3hY1258uL8ArOr80pohAAMwXVUnq7KGipqcE9TkfUej7NyzP1V7T3x5mIIyFADqo6xp5k2uJ9TkfUejzB0ZTtXeUxtKZaRADxiAeimj960JkyHiNHnf0SiXn3NyZA7Y5eec3N8NtmUyREI9k/DNbIGkWySdIOk5Sevc/XozWy3pdyXtDjf9kLt/JbzOKkkXSzos6VJ33xi2nyvpekkzJd3o7mu63TdJ+AAAVIdZkOmkScJPEoCdKOlEd/+2mf2MpM2SLpA0JunH7v7nHdufIumzks6UNFfSP0n6+fDf/0/SWZK2S7pX0q+5+3fj7psADAAANEWusyDdfZekXeHf/2FmD0nqFv6eL+lWd39W0uNm9qiCYEySHnX3x8KdvDXcNjYAAwAAGESpkvDNbKGk0yV9M2x6r5ltNbObzOy4sG2epMlTdLaHbXHtAAAArZI4ADOzF0r6gqT3u/uPJH1c0sslnaagh+wvJjaNuLp3ae+8n0vMbNzMxnfv3h1xFQDAwNi6Piguu3ok+F23Rd9Re7dv2aFlazZp0co7tGzNJt2+ZUfVu5RIolmQZjakIPj6tLvfJknu/oNJ//+EpC+HF7dLWjDp6vMl7Qz/jmt/nruvk7ROCnLAEj0KAED9dSZgN3lJKNTC7Vt26Ov/8DF9Trdq7tFPaee+43XdP1wo6fezTRYoQc8eMDMzSZ+U9JC7f3RS+4mTNvtlSRNrYGyQdKGZHW1miyQtlvQtBUn3i81skZkdJenCcFtgcFX57Z6eBdTJxDI0e7dJ8uD3+E3xldGBBO67Y52usnWaP+MpzTBp/oyndJWt0313rKt613pK0gO2TNK7JN1vZveFbR+S9GtmdpqCYcQnJP2eJLn7g2a2XkFy/SFJ73H3w5JkZu+VtFFBGYqb3P3BHB8LUC9VLprMgs2om6hlaCLX/VRrK6Mjvd858Hc6dsaBKW3H2gH9zoG/k/TH1exUQklmQX5d0flbX+lynQ9L+nBE+1e6XQ8YKFWue8aaa6ibNEFVXGX0Aa8hhfTmzvhhqvY6YSkioChVrnuW5r4ZqkQZYpeb6fh+H1cZPWoI80uXcry23E+HT0jVXicEYEBRqlz3LOl9c1JDWeLWsBz97WRrf3br1UVrHfvGq3Ro5jFT2g7NPEbHvrH+xwVrQQJFqXLds6T3zVAlypJ1Dcsqe5RRX0vHgkBm0nE1qyFD0wRgQFGqXDQ56X1zUqtGW3OZsiy0Pnt+2FMb0V6Gtr5mTZDluKoQARhQpCo/GJLcd9UntTZihmp/quxR5jVDAcgBA9osLi+njJNaW5HL1J+lY0F+WJJ8sbzxmqEA9IABbVblMGlbMezbv6p6lHnNUAACMKDtGpo/0VgM+zYPrxkKwBAkBgf1rNAEDPs2D69Z4zRhgW56wDAYSJJFUzRl2JdZf0c05TWDpCD4WnXb/dp/8LAkacee/Vp12/2SVKsFus09Zi2uGhgdHfXx8fGqdwNNsHZJzBDBAumyB6a3A4jX+YVGCnp8ykp6R/kGKOBetmaTduzpXHdUmjcyrHtWLi/0vs1ss7uPJtmWIUgMBpJkgfww669dBmxFjJ0RwVe39qoQgGEwVLnsDzBo+ELTLgMWcM8dGU7VXhUCMAwGkmSB/PCFpl0GLOC+/JyTNTw0c0rb8NBMXX7OyRXtUTQCMAyGKos0YrC1cXYtX2j618TjZcAC7gtOn6er33aq5o0MyxTkfl39tlNrlYAvkYQPAPHanIw+QEnZpWnq8dLU/a6hNEn4BGBAN5yE2o3ZtUijyccLn3W5SBOAUQcMiENtMQxYbgwKFnNc+N7tev2aTdq5Z7/mjgzr8nNOrt1wGCtilI8cMCDOgM0MQh8GLDcGBYs5Lnb6i7Vjz365jhQFrWNldpSLAAyIQ+9H8eqesEwyOtKIOF7262h95ODUnqX9Bw/r2o0Pl7lnqCECMCAOvR/FakLxR2bXIo2I42XlgYu14bnXT9u0bkVBUT5ywIA4K66InhlE70c+ug3x1inAITdmKpK1u+s4XsbXbJIigq26FQVF+egBA+I0qfej7kN5URjibZ4m9FrWTFOKgqJ89IC1Hd9mu2tC70cdZ2smOa5mz4+Zss8Qb201pdeyRiZmO1678eF6z4JE6QjA2qyOJ26kV7eTYtLjiiHe5qHXsi8XnD4vU8B1+5YdBHADiCHINqPMwmCo20kx6XHVpCFeBJiYkpvbt+zQsjWbtGjlHVq2ZlNsWYrbt+zQqtvup4zFAKIHrM3qduJGf+o2lJfmuGrCEC+OiOu1XHx2WAWeVIYkJoKq/QcPSzoSVEma1rN17caHn99uwkQZi6p6weiRywc9YG3Gt9nBULdaVRxXgyuq1/JV75S+8xkS81O4duPDOuvwv+jrR12qx45+p75+1KU66/C/RNYGiytXUVUZC3rk8kMA1mZ1O3GjP3UbyuO4ap40s2iXjgXrGq7eE/x+5B9JZUhp9Ed3as3QjZo/4ynNMGn+jKe0ZuhGjf7ozmnbxpWrqKqMRbceOaTDEGSbTZygmQXZfHUayuO4apask3FIZUht1VF/r2N1YErbsXZAHzxqvZatOWvK0N7l55w8ZbhSqraMRd165JqMAKzt6nTixuDguGqOrLNo65aD2AAv1VOR7Sf4D7UjDGQmhvauftupuvptp9Ym52ruyPDz+9jZjnQIwACgzbL2YFFOJDWLCVp3+ounXJ4Y2rtn5fLaJLnXrUeuycgBA4A2yzppom45iGlUtYJERJ7kPj9K1xya/pzVbWjvgtPn6eq3nap5I8MySfNGhnX1206tTYDYJD17wMxsgaRbJJ0g6TlJ69z9ejN7kaTPSVoo6QlJY+7+jJmZpOslvUnSPkm/6e7fDm/rIkl/FN70n7r7zfk+HABAKnn0YDVxyLnKQtQReZLX/ORXtOHZM6dtWsehvayFZRFIMgR5SNIfuvu3zexnJG02szsl/aaku9x9jZmtlLRS0gclvVHS4vDntZI+Lum1YcB2paRRSR7ezgZ3fybvBwUASKitkyaqXkGiI2g9bcsODVc4tEdtr/L1DMDcfZekXeHf/2FmD0maJ+l8SW8IN7tZ0j8rCMDOl3SLu7ukb5jZiJmdGG57p7s/LUlhEHeupM/m+HgAAGk1sQcrq5rN3qxyzcg0hWGrNkiBYqokfDNbKOl0Sd+U9NIwOJO77zKzl4SbzZM0Obtwe9gW1955H5dIukSSTjrppDS7BwBomMpOqDWcvVnW0F7nc77vwKHaVduXpu/nL75ijr6weUcjAsUkEgdgZvZCSV+Q9H53/1GQ6hW9aUSbd2mf2uC+TtI6SRodHZ32fwDAYEjb85JrsNbw2Zv9PhdRz3mcKicARO3np7/x/WlBQx0CxX4lmgVpZkMKgq9Pu/ttYfMPwqFFhb+fDNu3S1ow6erzJe3s0g4AaKE0VdVzXwKnwbM3szwXUc95nConAETtZ1yPTN1miiaVZBakSfqkpIfc/aOT/rVB0kWS1oS/vzip/b1mdquCJPy94RDlRkl/ZmbHhdudLWlVPg8DANA0aaqqF7Iodcbct6qGT7M8F0mDlapre6UJquo4UzSJJD1gyyS9S9JyM7sv/HmTgsDrLDN7RNJZ4WVJ+oqkxyQ9KukTkn5fksLk+z+RdG/4c9VEQj4AoH3SrHNYtyVwqlyUOstzEfecjwwP1aq2V9x+duYyxQaKVdV4SyHJLMivKzp/S5JWRGzvkt4Tc1s3SbopzQ4CAAZTmqrqdVsCp5AeuYSyPBdxz/nqt76ykgkAcb2Gcfv5K6+Zp7u/t7v79aus8ZYCSxEBACqRpvRC3ZbAqbJHLstz0ZRyF5n2s+oabwlZ0GFVT6Ojoz4+Pl71bgC9bV3fvkKWQFFi3k91qgG1bM2myF6okeEhveDoWYXvY52ei6RKe85Wjyg6Zd+k1Xv6v90EzGyzu48m2ZYeMAy+ooOjhnR3A43Q5f10weljtQkyonqhhmaYfnLgkPbsPyip2DpVTVwOKK53cM/+g/k+ZzWs8RaFxbgx2CY+zPduk+RHPszzTMjs1t0NIJ2GvJ+iFqV+4TGzdPDw1J6XuLIakRqQOJ5F0ny9VM9ZlIjFzutY440eMCTT1CG2MnIBarakCZBWrYazGvR+6uyFWrTyjsjtEuWFtaAnParXME6mXLqGrG9KAIbemvzBUMaHeUO6u4EotVsHsMHvp0wzNRuSOJ5FVGL9vgOH9My+g9O2zTy7tQHrmzIEid4aMiQQKe5DO88P84Z0dwNR0lSjT6Xf4bQGv58uP+dkDQ/NnNKWeKZmg3r+srjg9Hm6Z+VyPb7mzbpn5XJded4r+3/OGo4ADL01+YOhjA/zBi9pAhRSTiFL7mWD309ReWGJC5qW8WWxhjI9Zw3HECR6a/CQQCG5AHH5cA04QQCdCilwmnU4rcHvp75nJzZ8cfAsmjijMw8EYOit6R8MeX6YNzkfTmruZAoUppACp03uNY9R+ESFhiSOx6nVRI6GIABDbw3/YMhVkxNlmx48ohCFVEZvcq95hNImKjS05692Ezkagkr4QBplVljOu7dq7ZKYk+IC6bIH+r9doFNnsC8FveYNyeXqFFfBfd7IsO5Zubz3DQx4z3PVqwLUCZXwgaKU9c2+iN6qARwWyt2AnyhL05Be86TDZpkmKrSg57m0CvcDhlmQQBplTZEvovRHS2dZJVbGqgltsnQs6FldvSf4XbNgY2LYbMee/XIdCRBu37Jj2rZxExIy1/caEKVVuB8wBGBAGmVNkS+it6rB9ZVK0YITZRq3b9mhZWs2adHKO7RszabIwKTJ4uqfrd7w4LTHTX2v7qKenziZypsMGAIwIK0yvtkX0VvV4PpKpWjBiTKpNL1DTdVt2KzzcUuivlcXUbW8jjt2KHLbzBXuBwg5YEAdFVX6o6GzrEoxYDP3suhWHT8q6Mi7BEEZJQ3i6p91mnjc96xcTn2vLjpreXXOjJTaU+E+KXrAgDqit6p8DNE+L03Sed69ZWX1vpU2bNbS93KbK9wnRQ8YUFf0VpWroJl7TSxQmaY6ftresl7yvr04LAxdvLZWuE+KAAwAJuR8omxqgco01fHzXksy7e0lDXDjtmPYDFUhAAOQDbWzYpXVm5O3NNXx0/SWJQmW0t5ekgA36XaFrApQoib2trYZARiA/tWxyGSNAsK8e4fKlHT4KGlvWdIgKE3vW9IAN00g3NRhs6b2trYZSfgA+le32lk1K6aaqYBnQyRNtu4WBPVze1LyALfJgXBSSZ9f1Ac9YAD6V7faWTVbLD1Nb06TJek1ShMEJe2FihuunD08pGVrNj0/FDdy7FAxyfU10oYgc9DQAwagf3UrMlmzgLCOU/GrqnBfRG9gVCmJoRmmnxw4NKWMxY9/ekhDM23KdoMWCMc9jxPB6KCuaNBk9IAB6F/dikwWUEw1a2JznXKKqswTKqI3MGkpiYPPuUaGh/SCo2cNbIJ61PM7EYyyIHY9mbtXvQ+xRkdHfXx8vOrdANBNjZLep00KkIKAsM/Cl3FlCerQi9VPULhszabIIbt5I8O6Z+XyInZ1ijJm6S1aeYeizmom6fE1b871vuqm8/mNq2tW1ut4Et8nAAAUVklEQVTdRma22d1Hk2xLDxgwoU6BRJPUqchkzsVUeyU2VzHlP0svVtV5QmX0BqYpYzFoOp/fRSvviNyOvLB6IAADpHqWU0B/cgwI405UE0FPFUN5WWqLtSE4acvEhyTa8Ho3GUn4gFS/cgqohbgT1Uyzyqb8Z+nFikpaH7TgpI4TH6rShte7yegBA6TazZ5ri7pX7o7rTekMviaUMbSTpVej6ZXek6rTxIcqteX1bioCMEAqZPZcW/QbRDWhcnfcCezajQ/nPrST9HnMOsRGcNIuvN711TMAM7ObJL1F0pPuviRsWy3pdyXtDjf7kLt/JfzfKkkXSzos6VJ33xi2nyvpekkzJd3o7mvyfShABnUrp9AQWYKoOq6TmGTB5gl55hmleR7p1QAGQ5IesE9J+itJt3S0r3X3P5/cYGanSLpQ0islzZX0T2b28+G//1rSWZK2S7rXzDa4+3cz7DuQn5xnz7VFliCqqBl5ZfTI5R0EpX0em9CrUffhZaBqPQMwd/+amS1MeHvnS7rV3Z+V9LiZPSrpzPB/j7r7Y5JkZreG2xKAobeyykPUqZxCQ2QJooqYoVVmj1xUENRv0FF1eYi8NWF4GahallmQ7zWzrWZ2k5kdF7bNkzQ5kWZ72BbXDnRXs8WVMVWW5WWKmKGVZUHirEHQRNAxeQmcVbfdn2jplyYt2p1kKSMWhgZ66zcA+7ikl0s6TdIuSX8RtlvEtt6lfRozu8TMxs1sfPfu3VGboC62rpfWLpFWjwS/iwiK4spDfPWDxd83esoSRBVRLiBrj1ya9k5Zgo6mlAtIGmQOWo8eUIS+ZkG6+w8m/jazT0j6cnhxu6QFkzadL2ln+Hdce+dtr5O0TgqWIupn/1CCsgqXxpWB2P908FPkfddQ3fJqsuZC5Z3LlGVYM+vswixBR1MS65MO01IAFOitrwDMzE50913hxV+W9ED49wZJnzGzjypIwl8s6VsKesAWm9kiSTsUJOq/M8uOD5K6nVQT6Va4NM8gKK48RKci7rtmysyrSXNM5h1EZXk/ZAmisgZBWYOOJiTWJw0yqUYP9JakDMVnJb1B0vFmtl3SlZLeYGanKRhGfELS70mSuz9oZusVJNcfkvQedz8c3s57JW1UUIbiJnd/MPdH00CNTVYtq3BpVHmItPs0IMoq21DlMZn1vqvskWtD0JE0yGxKjx5QJXOv7yjf6Oioj4+PV70bhVq2ZlPkB1rtV6tfuySmcOkC6bIHprdn0TkL8sBPjgw/Fn3fNbJo5R2RiZMm6fE1b87tfqo8Jhv7fgjl3Ztdt97xzgBZCoLMti71A3Qys83uPppkWyrhV6yIZNVSPrSLKlwaV3Ji8tBiZ/5ZeN/3vvx9ev+aTbU5WeWtrLyaKhOom568necwYh17x+nZAvJDAFaxvE+qpX1o51C4tDNQvO6UR3TG/Vf2TuyPuO97X/4+vfvel2l/eN06nKzyVtYQV5UJ1CRvH1HHlQKkZuSqAU2QpQ4YcpD39PNS6+8sHQuG/FbvCX4vHUtcmiJqOvvczdfEJ/b3uO/3f3fxwNcd6la2IUltpqSqLInQlHIMZWh6byCA7ugBq1jeXfqVfminKE0RFSieqKeibzdBcn3TT1ZJh43jqq8n7fVMcj9xx6QU5GgVOfTEENcR9AYCg40ArAby7NJP+6Gda75YitIUUYHRTj9e8y0iCJs9v+ddN/lklXXYOOlQVdq1Dvu9blZlDXHVLcG9UxtmVQJtxhDkgEkzhJNl6ZRIKUpTRAVG1xwa034dPbUxYWJ/k4eusg4bJ+39y3I/g7a0TO7HfgGKWCkAQH3QAzZg0gzh5J7kG1c0NaIHK+rb/Z0z/5ve9eqFOuNf/zJ1Yn+Th66yDp8m7f3Lcj9VD/Fm7a3qvP6+A4dqmeDeKU1vYN179ABMRQDWRVM/0JJ+aOd+Uk1RmiIuYDrj9HMV1vVNramzs+ICqNnDQ4lyrpIOVWUZpq1yiDfr8GfU9eM0JWewUx1LVgDojgAsRqkfaHG1rwqW+0l16ZjufeIZLfj2tXqJP6Un7XhtO/VynRHzWLIETE0NjqNEBVBDM0w/OXBIe/YflNQ7X0vq3fuXJacoj3ykfl+zrD21UdePU1bOYN7Hb11LVgCIRwAWo6gPtL5rXxUgzUk1yQnj9i07tOrel2n/weuP3N69M3X1gh2NWCqnqqAuKoDad+CQntl3cMp23Y6/JMFslmHarEO8WV6zbj21SV6zpL1aZeUMFnH8Vj1EDCA9ArAYRVWo7/zgnbv5GslKWNQ6QtKTatITRlnfwou4n6qHcDoDqEUr74jcLusJNUuvY5brZnnNug3RJnnN4q4/MjykFxw9q/SAu4jjt8mzgIG2IgCLkSYvR+o/6T1L7as8JDmpJj1hFPUtvLOXIy6HJ8v91G0IZ9BOqFmOjbieWjMles3irr/6ra+s5LUt4n1CyQqgeQjAQp0n+V98xRx9YfOOnnk5l//9dySTDh7259viek7yrn1VlH4DniKChqieKZMiF6VOU+tMUuFBXRaDdkLNcmzE9dRe9rn7IrfvfM3qNkO2iPdJ3R4jgN4IwBR9kv/C5h36ldfM093f2901L+fgc9NDgbiek6gP3msOjekjR31Sw3r2SGOaRa1zTuDPEvAUETRE9Uy5NG2fetU6m/x4ooLmtEFd0fI4odZpokLWYyOqp/bajQ8nDmTqNEO2qOC6To8RQG8EYIoffrr7e7t1z8rlz7fF5eVEieo5ybv2lbau16Evvk+zDv80uLx3W3BZ6jsIyxLwFPEtPK4HyhUUpuxn2DcqaE4T1JUl6yzRPJcnyqqIY6OpvYT0VgGQCMAkJc/J6DZU1albDad+a191nii/fPiPdNxE8BWadfin2vfVK3RsVACWoLcsTcAjRa8NmOeJJO45nzcyPCU4jpNmCDFpUNcERSxPlFXex0aTAxl6qwAQgCl5TkZcvabJw1kTbd1qOPXzwRt1opx99A+CbpsOx+z/9+mNCRfKThrwpD1x99vLkrWXI03QnDSoa4I8lidqQoBAIAOgqVgLUsnXEYxam+3aX32Vrn37q6a0vfCYWVMCMin7unlRJ8qdfnzktjufe/H0xm4LZU+S9LlIszZglnX3sq6HF/V4hmaYhmZOjVyzDl3dvmWHlq3ZpEUr79CyNZsqX1MwLnctz+WJAAD9owdM6YYy4r5xF13DKeq61xwa05qhG3WsHXi+bZ8fpRuP+g2t7tw44ULZF5w+T/O2fTmsZr9bT9ocbXv15eEwaff9iWvP2suStXbVxD70UzokSq8Zs3VYBqaM5YlQf3WaiAFgKgKwUJ5DGUWc1KJuc8Nzr5cdlC6ftV5z7Yfa6S/WdbpQr3/zJdOuv2/4BB27f1d0++SGreuDyvzaL5l0gnbrhPuvlBYel2ioMuoxVt3LkiRoTipq6PXT3/j+tBmUVQ/jlbE8Eeqt6uLCALojACtAEevmRdUlGx6aqZ95zTv1ju+d1fMb7jUH36EP+Mem9ZZdc/AdU3vLug1VTgrA0jzGNMFa3b+xx80SjVL1MF7RyxOh3pqe3wcMOgKwAhSxbl5UXbI0t3nzj8/U0zMO6AOTesuuOTSmLz175tQALMVQZdLHmDRYq+M39qRFaaPMHRlOVAS26oCHRPbBVHXPM4DuCMAKUsS6eZ11ydKYOzKsDXterw0HXj+lfV5nL9Ts+cEMyU4RlfmTPsakeWV1+8aepihtVA2xX3zFnERFYKsOMotQ957MNiC/D6g3ZkGWbet6ae0SafVI8Hvr+mmbFLVWXJLZjVpxRVCJf7I0lfmjhHllJ2i3ZoR5ZWfcf+W0x163b+zditJONjw0U7/+upOmzdS8+3u7I4vA5j1Dtm6yzHpFfhK/5wFUgh6wMmWsxVXKWnET+5Hj8kZJ88rq9o09axX+uLUK09xXE9WtJ7OtyO8D6o0ArEwFJLinkXhYdOlYtoCrU2xe2bagFzAM9K475X16970vq82MvKxV+NPkjA3SsFDdejLbjPw+oL4YgixTigT3yOKjM+/pOXxZSxH5YwEL881c2rtNZ9x/pW4549/6Lrqat6xDOGUVga2bpEVgAaDN6AErU5YE94TDl7W04oqp+y5petq6pIP7dca//qXuWflAmXsXK+sQThFFYJuA2mIA0Ju5x1Uxqt7o6KiPj49XvRv56QyipCDB/bwbegdRa5fEBG8LpMs6ApYEi26XrnOfoh6LJMmk1XtK3TXkj1mQANrIzDa7+2iSbekBK1OWBPeEw5e17SnrzCuLDSjjhivRJOQeAUB3BGBl6zfBPenwZcJE/8pFDUtmLXcBAEBDkISfVoI6XoVIWp8raU9Z1ZaOBUOvsxdIsuB3kqFYAAAGQM8eMDO7SdJbJD3p7kvCthdJ+pykhZKekDTm7s+YmUm6XtKbJO2T9Jvu/u3wOhdJ+qPwZv/U3W/O96GUoMrhvaTDlykS/SuXd7kLAAAaIkkP2KckndvRtlLSXe6+WNJd4WVJeqOkxeHPJZI+Lj0fsF0p6bWSzpR0pZkdl3XnS9dteK8MS8eChPvVe4LfUcFLEZXsAQBArnoGYO7+NUlPdzSfL2miB+tmSRdMar/FA9+QNGJmJ0o6R9Kd7v60uz8j6U5ND+rqr8zhvX6HOhnaAwCg9vpNwn+pu++SJHffZWYvCdvnSZo8/rU9bItrb5ayhveyDnUytAcAQK3lnYTfuU6xFL1+8UT79Bswu8TMxs1sfPfu3bnuXGZlDe9VPdQJAAAK1W8A9oNwaFHh7yfD9u2SFkzabr6knV3ap3H3de4+6u6jc+bM6XP3ClLW8F5TZjICAIC+9DsEuUHSRZLWhL+/OKn9vWZ2q4KE+73hEOVGSX82KfH+bEmr+t/tCpUxvNekmYwAACC1nj1gZvZZSf9X0slmtt3MLlYQeJ1lZo9IOiu8LElfkfSYpEclfULS70uSuz8t6U8k3Rv+XBW2IQozGQEAGGisBVmUrOsx1nE9RwAAEIu1IKuWR8FWZjICADCwWIqoCMxiBAAAXRCAFYFZjAAAoAsCsAl5LrIdN1uRWYwAAEAEYIGJnK292yT5kZytfoMwZjECAIAuCMCk/HO2WI8RAAB0wSxIqZicLWYxAgCAGPSASeRsAQCAUhGASeRsAQCAUhGASeRsAQCAUpEDNoGcLQAAUBJ6wAAAAEpGAAYAAFAyArBu8qyODwAAECIHLM5EdfyJAq0T1fElcsUAAEAm9IDFybs6PgAAQIgALE6a6vgMVQIAgBQIwOIkrY6f90LeAABg4BGAxUlaHZ+hSgAAkBIBWJyk1fGLWMgbAAAMNGZBdpOkOv7s+eHwY0Q7AABABHrAsmIhbwAAkBIBWFYs5A0AAFJiCDIPLOQNAABSoAcMAACgZARgAAAAJSMAAwAAKBkBGAAAQMkIwAAAAEpGAAYAAFAyAjAAAICSEYABAACUjAAMAACgZARgAAAAJSMAAwAAKJm5e9X7EMvMdkv6t5Lv9nhJT5V8n0iG16beeH3qi9emvnht6i3t6/Myd5+TZMNaB2BVMLNxdx+tej8wHa9NvfH61BevTX3x2tRbka8PQ5AAAAAlIwADAAAoGQHYdOuq3gHE4rWpN16f+uK1qS9em3or7PUhBwwAAKBk9IABAACUjAAsZGbnmtnDZvaoma2sen/azswWmNndZvaQmT1oZn8Qtr/IzO40s0fC38dVva9tZWYzzWyLmX05vLzIzL4ZvjafM7Ojqt7HNjKzETP7vJl9L3z//GfeN/VhZpeFn2kPmNlnzewY3jvVMLObzOxJM3tgUlvke8UCN4QxwlYze3XW+ycAU3AikfTXkt4o6RRJv2Zmp1S7V613SNIfuvsvSHqdpPeEr8lKSXe5+2JJd4WXUY0/kPTQpMsfkbQ2fG2ekXRxJXuF6yX9b3d/haRXKXiNeN/UgJnNk3SppFF3XyJppqQLxXunKp+SdG5HW9x75Y2SFoc/l0j6eNY7JwALnCnpUXd/zN0PSLpV0vkV71Orufsud/92+Pd/KDiJzFPwutwcbnazpAuq2cN2M7P5kt4s6cbwsklaLunz4Sa8NhUws/8k6b9K+qQkufsBd98j3jd1MkvSsJnNknSspF3ivVMJd/+apKc7muPeK+dLusUD35A0YmYnZrl/ArDAPEnbJl3eHrahBsxsoaTTJX1T0kvdfZcUBGmSXlLdnrXadZI+IOm58PKLJe1x90PhZd5D1fhZSbsl/U04PHyjmb1AvG9qwd13SPpzSd9XEHjtlbRZvHfqJO69knucQAAWsIg2pofWgJm9UNIXJL3f3X9U9f5AMrO3SHrS3TdPbo7YlPdQ+WZJerWkj7v76ZJ+IoYbayPMJzpf0iJJcyW9QMHQVifeO/WT+2ccAVhgu6QFky7Pl7Szon1ByMyGFARfn3b328LmH0x0+4a/n6xq/1psmaS3mtkTCobrlyvoERsJh1Uk3kNV2S5pu7t/M7z8eQUBGe+bevglSY+7+253PyjpNkn/Rbx36iTuvZJ7nEAAFrhX0uJwJspRCpIiN1S8T60W5hR9UtJD7v7RSf/aIOmi8O+LJH2x7H1rO3df5e7z3X2hgvfKJnf/dUl3S3p7uBmvTQXc/d8lbTOzk8OmFZK+K943dfF9Sa8zs2PDz7iJ14f3Tn3EvVc2SHp3OBvydZL2TgxV9otCrCEze5OCb/EzJd3k7h+ueJdazcxeL+n/SLpfR/KMPqQgD2y9pJMUfJj9qrt3JlGiJGb2Bkn/093fYmY/q6BH7EWStkj6DXd/tsr9ayMzO03B5IijJD0m6bcUfNnmfVMDZvbHkt6hYKb3Fkm/oyCXiPdOyczss5LeIOl4ST+QdKWk2xXxXgkD5r9SMGtyn6TfcvfxTPdPAAYAAFAuhiABAABKRgAGAABQMgIwAACAkhGAAQAAlIwADAAAoGQEYAAAACUjAAMAACgZARgAAEDJ/j9iJuGkjOe7gQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def getMovingAve(movingAveSize, scores):\n",
    "    movingAve = []\n",
    "    \n",
    "    for i in range(len(scores) // movingAveSize):\n",
    "        if (i + 1) * movingAveSize < len(scores):\n",
    "            movingAve.append(np.mean(scores[i*movingAveSize:(i+1)*movingAveSize]))\n",
    "    return movingAve\n",
    "\n",
    "\n",
    "def mapScores():\n",
    "    movingAveSize = 5\n",
    "    \n",
    "    movingAveA = getMovingAve(movingAveSize*5, gameScoreLog)\n",
    "    movingAveB = getMovingAve(movingAveSize, greedyScoreLog)\n",
    "    \n",
    "    \n",
    "    plt.figure(figsize=(10,6))\n",
    "    plt.scatter(np.arange(len(movingAveA)),movingAveA)\n",
    "    plt.scatter(np.arange(len(movingAveB)),movingAveB)\n",
    "    \n",
    "mapScores()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Score:  2827.712\n",
      "Std Deviation:  1104.88\n",
      "Time elapsed:   552.85\n"
     ]
    }
   ],
   "source": [
    "def testAgent():\n",
    "    games = 500\n",
    "    \n",
    "    scores = np.zeros(games)\n",
    "    \n",
    "    for i in range(games):\n",
    "        scores[i] = (run(initializeEnv(),agent,memCount,True))\n",
    "    \n",
    "    print(\"Average Score: \", np.mean(scores))\n",
    "    print(\"Std Deviation: \", format(np.std(scores),'.2f'))\n",
    "    print(\"Time elapsed:  \", format(time.time() - startTime, '.2f'))\n",
    "    \n",
    "\n",
    "testAgent()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}