Deep Q - Vanilla.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intitializeAgent():\n",
    "    \n",
    "    model = Sequential() \n",
    "    \n",
    "    model.add(Dense(256,activation='relu',input_shape=(INPUT_SIZE,)))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(256,activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(256,activation='relu'))\n",
    "    model.add(Dense(OUTPUT_SIZE,activation='linear'))\n",
    "    \n",
    "    model.compile(optimizer='adam',loss='mse',metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getPrediction(env,agent):\n",
    "    env = env.reshape(INPUT_SIZE,1).reshape(1,-1)\n",
    "    return agent.predict(env)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAction(output,invalidActions):\n",
    "    rand = np.random.rand()\n",
    "    output[invalidActions] = np.min(output) - 0.1\n",
    "    \n",
    "    if rand < 1 - EPSILON_RATE * iteration:\n",
    "        action = np.random.randint(4)\n",
    "        while action in invalidActions:\n",
    "            action = np.random.randint(4)\n",
    "        return action\n",
    "    else:\n",
    "        return np.argmax(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getGreedyAction(output,invalidActions):\n",
    "    output[invalidActions] = np.min(output) - 0.1\n",
    "    return np.argmax(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(initialStateMemory,actionMemory,rewardMemory,finalStateMemory,model,epochs):\n",
    "    \n",
    "    if memCount < MAX_MEM_SIZE:\n",
    "        sample = np.random.choice(memCount, SAMPLE_SIZE)\n",
    "    else:\n",
    "        sample = np.random.choice(MAX_MEM_SIZE, SAMPLE_SIZE)\n",
    "    batchSize = sample.shape[0] // BATCH_SIZE\n",
    "    \n",
    "    if batchSize == 0:\n",
    "        batchSize = 1\n",
    "                \n",
    "    targetQs = getTargetQs(initialStateMemory[sample],\n",
    "                            actionMemory[sample],\n",
    "                            rewardMemory[sample],\n",
    "                            finalStateMemory[sample],\n",
    "                            model)\n",
    "        \n",
    "    print(\"Training Model...\")\n",
    "    model.fit(initialStateMemory[sample],targetQs,batch_size = batchSize, epochs = epochs, verbose = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Replay Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addMemory(initialState, action, reward, finalState, memCount, MAX_MEM_SIZE):\n",
    "    if memCount < MAX_MEM_SIZE:\n",
    "        initialStateMemory[memCount] = initialState.reshape(INPUT_SIZE)\n",
    "        actionMemory[memCount] = action\n",
    "        rewardMemory[memCount] = reward\n",
    "        finalStateMemory[memCount] = finalState.reshape(INPUT_SIZE)\n",
    "    else:\n",
    "        rand = np.random.randint(MAX_MEM_SIZE)\n",
    "        initialStateMemory[rand] = initialState.reshape(INPUT_SIZE)\n",
    "        actionMemory[rand] = action\n",
    "        rewardMemory[rand] = reward\n",
    "        finalStateMemory[rand] = finalState.reshape(INPUT_SIZE)\n",
    "        \n",
    "    memCount += 1\n",
    "    \n",
    "    return initialStateMemory,actionMemory,rewardMemory,finalStateMemory,memCount"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTargetQs(states0, actions0, rewards1, states1, agent):\n",
    "    DISCOUNT_FACTOR = 0.99\n",
    "    targetQs = np.zeros((len(states0),OUTPUT_SIZE))\n",
    "    \n",
    "    for i in range(len(states0)):\n",
    "        targetQs[i] = getPrediction(states0[i],agent)\n",
    "        targetQs[i][int(actions0[i])] = rewards1[i] + DISCOUNT_FACTOR * np.max(getPrediction(states1[i],agent))\n",
    "    return targetQs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Game Logic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initializeEnv():\n",
    "    env = np.zeros((BOARD_LENGTH,BOARD_LENGTH))\n",
    "\n",
    "    for i in range(2):\n",
    "        addValue(env)\n",
    "            \n",
    "    return env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addValue(env):\n",
    "#     rand = np.random.rand()\n",
    "#     if rand > 0.1:\n",
    "        # value = 2\n",
    "#     else:\n",
    "#         value = 4\n",
    "    \n",
    "    value = 2\n",
    "        \n",
    "    coordinate1 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "    coordinate2 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "        \n",
    "    if env[coordinate1,coordinate2] != 0:\n",
    "        getNewCoordinate(env,value)\n",
    "    else:\n",
    "        env[coordinate1,coordinate2] = value\n",
    "        \n",
    "    return env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getNewCoordinate(env,value):\n",
    "    \n",
    "    coordinate1 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "    coordinate2 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "    \n",
    "    while env[coordinate1,coordinate2] != 0:\n",
    "        coordinate1 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "        coordinate2 = random.sample(range(0,BOARD_LENGTH),1)\n",
    "    \n",
    "    env[coordinate1,coordinate2] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def step(env,action,score):\n",
    "    \n",
    "    if action == 0:\n",
    "        env, score, validAction = actionUp(env,score)\n",
    "    if action == 1:\n",
    "        env, score, validAction = actionDown(env,score)\n",
    "    if action == 2:\n",
    "        env, score, validAction = actionLeft(env,score)\n",
    "    if action == 3:\n",
    "        env, score, validAction = actionRight(env,score)\n",
    "    \n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actionUp(env,score):\n",
    "    validAction = False\n",
    "    \n",
    "    for i in range(BOARD_LENGTH):\n",
    "        for j in range(BOARD_LENGTH):\n",
    "            row = i\n",
    "            while env[row][j] != 0 and row != 0:\n",
    "                if env[row - 1][j] != 0:\n",
    "                    if env[row - 1][j] == env[row][j]:\n",
    "                        env[row - 1][j] *= 2\n",
    "                        env[row][j] = 0\n",
    "                        score += env[row - 1][j]\n",
    "                        validAction = True\n",
    "                    break\n",
    "                temp = env[row][j]\n",
    "                env[row-1][j] = temp\n",
    "                env[row][j] = 0\n",
    "                row -= 1\n",
    "                validAction = True\n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actionDown(env,score):\n",
    "    validAction = False\n",
    "    \n",
    "    for i in range(3,-1,-1):\n",
    "        for j in range(BOARD_LENGTH):\n",
    "            row = i\n",
    "            while env[row][j] != 0 and row != 3:\n",
    "                if env[row + 1][j] != 0:\n",
    "                    if env[row + 1][j] == env[row][j]:\n",
    "                        env[row + 1][j] *= 2\n",
    "                        env[row][j] = 0\n",
    "                        score += env[row + 1][j]\n",
    "                        validAction = True\n",
    "                    break\n",
    "                temp = env[row][j]\n",
    "                env[row + 1][j] = temp\n",
    "                env[row][j] = 0\n",
    "                row += 1\n",
    "                validAction = True\n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actionLeft(env,score):\n",
    "    validAction = False\n",
    "    env = env.T\n",
    "    \n",
    "    for i in range(BOARD_LENGTH):\n",
    "        for j in range(BOARD_LENGTH):\n",
    "            row = i\n",
    "            while env[row][j] != 0 and row != 0:\n",
    "                if env[row - 1][j] != 0:\n",
    "                    if env[row - 1][j] == env[row][j]:\n",
    "                        env[row - 1][j] *= 2\n",
    "                        env[row][j] = 0\n",
    "                        score += env[row - 1][j]\n",
    "                        validAction = True\n",
    "                    break\n",
    "                temp = env[row][j]\n",
    "                env[row-1][j] = temp\n",
    "                env[row][j] = 0\n",
    "                row -= 1\n",
    "                validAction = True\n",
    "    env = env.T\n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actionRight(env,score):\n",
    "    validAction = False\n",
    "    env = env.T\n",
    "    \n",
    "    for i in range(3,-1,-1):\n",
    "        for j in range(BOARD_LENGTH):\n",
    "            row = i\n",
    "            while env[row][j] != 0 and row != 3:\n",
    "                if env[row + 1][j] != 0:\n",
    "                    if env[row + 1][j] == env[row][j]:\n",
    "                        env[row + 1][j] *= 2\n",
    "                        env[row][j] = 0\n",
    "                        score += env[row + 1][j]\n",
    "                        validAction = True\n",
    "                    break\n",
    "                temp = env[row][j]\n",
    "                env[row + 1][j] = temp\n",
    "                env[row][j] = 0\n",
    "                row += 1\n",
    "                validAction = True\n",
    "    env = env.T\n",
    "    return env, score, validAction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAvailableAction(env):\n",
    "    tempEnv = np.copy(env)\n",
    "    available = False\n",
    "    \n",
    "    for i in range(tempEnv.shape[0]):\n",
    "        for j in range(tempEnv.shape[0] - 1):\n",
    "            if tempEnv[i][j] == tempEnv[i][j+1]:\n",
    "                available = True\n",
    "                \n",
    "                \n",
    "    tempEnv = tempEnv.T\n",
    "    for i in range(tempEnv.shape[0]):\n",
    "        for j in range(tempEnv.shape[0] - 1):\n",
    "            if tempEnv[i][j] == tempEnv[i][j+1]:\n",
    "                available = True\n",
    "                break\n",
    "    \n",
    "    return available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(env,agent,memCount,testFlag,score = 0):\n",
    "    \n",
    "    gameOver = False\n",
    "    while gameOver == False:\n",
    "        invalidActions = []\n",
    "        validAction = False\n",
    "        while validAction == False:\n",
    "            \n",
    "            if not testFlag:\n",
    "                previousEnv = np.copy(env)\n",
    "                previousScore = np.copy(score)\n",
    "            \n",
    "            output = getPrediction(env,agent)\n",
    "            if testFlag: \n",
    "                action = getGreedyAction(output,invalidActions) \n",
    "            else: \n",
    "                action = getAction(output,invalidActions)\n",
    "            invalidActions.append(action)\n",
    "            env, score, validAction = step(env, action, score)\n",
    "        \n",
    "        if not testFlag:\n",
    "            initialStateMemory,actionMemory,rewardMemory,finalStateMemory,memCount = addMemory(previousEnv, action, score - previousScore, env, memCount, MAX_MEM_SIZE)\n",
    "\n",
    "        env = addValue(env)\n",
    "        \n",
    "        if np.sum(env == 0) == 0:\n",
    "            gameOver = not getAvailableAction(env)\n",
    "    \n",
    "    if not testFlag:\n",
    "        return score,initialStateMemory,actionMemory,rewardMemory,finalStateMemory,memCount\n",
    "    else: \n",
    "        return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main Output Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printIter():\n",
    "    print(\"\\n === === === Iteration\",iteration,\"of\",ITERATIONS,'=== === ===\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printStats(timeLog,iteration,ITERATIONS,memCount,scoreLog,greedyScoreLog,EPISODES,TESTS):\n",
    "    print(\"\\nEstimated time remaining: \", format(np.mean(timeLog) * (ITERATIONS - iteration) / 60,'.2f'),\" Minutes\\n\")\n",
    "    print(\"Memories Elapsed: \",memCount)\n",
    "    print(\"Current Epsilon: \",format(1 - EPSILON_RATE * iteration,'.4f'))\n",
    "    print(\"Ave Score for Epoch: \", format(np.mean(scoreLog[len(scoreLog) - EPISODES : len(scoreLog)-1]) ,'.2f'))\n",
    "    print(\"Ave Greedy Score for Epoch: \", format(np.mean(greedyScoreLog[len(greedyScoreLog) - TESTS : len(greedyScoreLog)-1]) ,'.2f'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printTrainInfo():\n",
    "    print(\"Maximum memory size:\",MAX_MEM_SIZE)\n",
    "    print(\"Sample size for training:\",SAMPLE_SIZE)\n",
    "    print(\"Batch size from sample:\",BATCH_SIZE)\n",
    "    print(\"Epslon Rate:\",format(EPSILON_RATE,'.4f'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import time\n",
    "import sys\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras import layers\n",
    "from keras.layers import Dense, Dropout, Conv2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0509 13:44:27.807257 10800 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W0509 13:44:27.837109 10800 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0509 13:44:27.844098 10800 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W0509 13:44:27.858339 10800 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:133: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "W0509 13:44:27.864323 10800 deprecation.py:506] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0509 13:44:27.914950 10800 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "BOARD_LENGTH = 4\n",
    "INPUT_SIZE = 16\n",
    "OUTPUT_SIZE = BOARD_LENGTH\n",
    "\n",
    "memCount = 0\n",
    "MAX_MEM_SIZE = 100_000\n",
    "SAMPLE_SIZE = MAX_MEM_SIZE // 200\n",
    "BATCH_SIZE = SAMPLE_SIZE // 100\n",
    "initialStateMemory = np.zeros((MAX_MEM_SIZE,INPUT_SIZE))\n",
    "actionMemory = np.zeros(MAX_MEM_SIZE)\n",
    "rewardMemory = np.zeros(MAX_MEM_SIZE)\n",
    "finalStateMemory = np.zeros((MAX_MEM_SIZE,INPUT_SIZE))\n",
    "\n",
    "\n",
    "gameScoreLog = []\n",
    "greedyScoreLog = []\n",
    "agent = intitializeAgent()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "ITERATIONS = 100\n",
    "EPISODES = 100\n",
    "TESTS = EPISODES // 5\n",
    "EPOCHS = 4\n",
    "MIN_EPSILON = 0.01\n",
    "EPSILON_RATE = (1 - MIN_EPSILON) / ITERATIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0509 13:44:27.964992 10800 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:2741: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum memory size: 100000\n",
      "Sample size for training: 500\n",
      "Batch size from sample: 5\n",
      "Epslon Rate: 0.0099\n",
      "\n",
      " === === === Iteration 0 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  22.29  Minutes\n",
      "\n",
      "Memories Elapsed:  14141\n",
      "Current Epsilon:  1.0000\n",
      "Ave Score for Epoch:  1272.08\n",
      "Ave Greedy Score for Epoch:  1568.42\n",
      "\n",
      " === === === Iteration 1 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.83  Minutes\n",
      "\n",
      "Memories Elapsed:  17498\n",
      "Current Epsilon:  0.9901\n",
      "Ave Score for Epoch:  1216.33\n",
      "Ave Greedy Score for Epoch:  785.00\n",
      "\n",
      " === === === Iteration 2 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.02  Minutes\n",
      "\n",
      "Memories Elapsed:  20715\n",
      "Current Epsilon:  0.9802\n",
      "Ave Score for Epoch:  1079.83\n",
      "Ave Greedy Score for Epoch:  716.00\n",
      "\n",
      " === === === Iteration 3 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.87  Minutes\n",
      "\n",
      "Memories Elapsed:  24088\n",
      "Current Epsilon:  0.9703\n",
      "Ave Score for Epoch:  1180.17\n",
      "Ave Greedy Score for Epoch:  1346.00\n",
      "\n",
      " === === === Iteration 4 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.06  Minutes\n",
      "\n",
      "Memories Elapsed:  27807\n",
      "Current Epsilon:  0.9604\n",
      "Ave Score for Epoch:  1370.00\n",
      "Ave Greedy Score for Epoch:  1550.00\n",
      "\n",
      " === === === Iteration 5 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.11  Minutes\n",
      "\n",
      "Memories Elapsed:  30759\n",
      "Current Epsilon:  0.9505\n",
      "Ave Score for Epoch:  1015.83\n",
      "Ave Greedy Score for Epoch:  2286.00\n",
      "\n",
      " === === === Iteration 6 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  5.11  Minutes\n",
      "\n",
      "Memories Elapsed:  34301\n",
      "Current Epsilon:  0.9406\n",
      "Ave Score for Epoch:  1248.67\n",
      "Ave Greedy Score for Epoch:  1504.00\n",
      "\n",
      " === === === Iteration 7 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.96  Minutes\n",
      "\n",
      "Memories Elapsed:  37527\n",
      "Current Epsilon:  0.9307\n",
      "Ave Score for Epoch:  1071.83\n",
      "Ave Greedy Score for Epoch:  2035.00\n",
      "\n",
      " === === === Iteration 8 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.86  Minutes\n",
      "\n",
      "Memories Elapsed:  41255\n",
      "Current Epsilon:  0.9208\n",
      "Ave Score for Epoch:  1348.17\n",
      "Ave Greedy Score for Epoch:  2178.00\n",
      "\n",
      " === === === Iteration 9 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.77  Minutes\n",
      "\n",
      "Memories Elapsed:  44910\n",
      "Current Epsilon:  0.9109\n",
      "Ave Score for Epoch:  1339.33\n",
      "Ave Greedy Score for Epoch:  1431.00\n",
      "\n",
      " === === === Iteration 10 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.68  Minutes\n",
      "\n",
      "Memories Elapsed:  48497\n",
      "Current Epsilon:  0.9010\n",
      "Ave Score for Epoch:  1335.00\n",
      "Ave Greedy Score for Epoch:  1206.00\n",
      "\n",
      " === === === Iteration 11 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.59  Minutes\n",
      "\n",
      "Memories Elapsed:  51874\n",
      "Current Epsilon:  0.8911\n",
      "Ave Score for Epoch:  1195.83\n",
      "Ave Greedy Score for Epoch:  2385.00\n",
      "\n",
      " === === === Iteration 12 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.50  Minutes\n",
      "\n",
      "Memories Elapsed:  55300\n",
      "Current Epsilon:  0.8812\n",
      "Ave Score for Epoch:  1162.83\n",
      "Ave Greedy Score for Epoch:  1747.00\n",
      "\n",
      " === === === Iteration 13 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.43  Minutes\n",
      "\n",
      "Memories Elapsed:  58817\n",
      "Current Epsilon:  0.8713\n",
      "Ave Score for Epoch:  1239.50\n",
      "Ave Greedy Score for Epoch:  1084.00\n",
      "\n",
      " === === === Iteration 14 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.44  Minutes\n",
      "\n",
      "Memories Elapsed:  62288\n",
      "Current Epsilon:  0.8614\n",
      "Ave Score for Epoch:  1284.17\n",
      "Ave Greedy Score for Epoch:  1359.00\n",
      "\n",
      " === === === Iteration 15 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.38  Minutes\n",
      "\n",
      "Memories Elapsed:  65898\n",
      "Current Epsilon:  0.8515\n",
      "Ave Score for Epoch:  1371.17\n",
      "Ave Greedy Score for Epoch:  987.00\n",
      "\n",
      " === === === Iteration 16 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.31  Minutes\n",
      "\n",
      "Memories Elapsed:  69269\n",
      "Current Epsilon:  0.8416\n",
      "Ave Score for Epoch:  1239.33\n",
      "Ave Greedy Score for Epoch:  1287.00\n",
      "\n",
      " === === === Iteration 17 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.27  Minutes\n",
      "\n",
      "Memories Elapsed:  72948\n",
      "Current Epsilon:  0.8317\n",
      "Ave Score for Epoch:  1302.83\n",
      "Ave Greedy Score for Epoch:  924.00\n",
      "\n",
      " === === === Iteration 18 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.19  Minutes\n",
      "\n",
      "Memories Elapsed:  76468\n",
      "Current Epsilon:  0.8218\n",
      "Ave Score for Epoch:  1260.17\n",
      "Ave Greedy Score for Epoch:  770.00\n",
      "\n",
      " === === === Iteration 19 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.11  Minutes\n",
      "\n",
      "Memories Elapsed:  79720\n",
      "Current Epsilon:  0.8119\n",
      "Ave Score for Epoch:  1118.50\n",
      "Ave Greedy Score for Epoch:  1914.00\n",
      "\n",
      " === === === Iteration 20 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.07  Minutes\n",
      "\n",
      "Memories Elapsed:  83358\n",
      "Current Epsilon:  0.8020\n",
      "Ave Score for Epoch:  1325.00\n",
      "Ave Greedy Score for Epoch:  1054.00\n",
      "\n",
      " === === === Iteration 21 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.06  Minutes\n",
      "\n",
      "Memories Elapsed:  87082\n",
      "Current Epsilon:  0.7921\n",
      "Ave Score for Epoch:  1325.67\n",
      "Ave Greedy Score for Epoch:  636.00\n",
      "\n",
      " === === === Iteration 22 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  4.01  Minutes\n",
      "\n",
      "Memories Elapsed:  90182\n",
      "Current Epsilon:  0.7822\n",
      "Ave Score for Epoch:  1047.83\n",
      "Ave Greedy Score for Epoch:  520.00\n",
      "\n",
      " === === === Iteration 23 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.96  Minutes\n",
      "\n",
      "Memories Elapsed:  93609\n",
      "Current Epsilon:  0.7723\n",
      "Ave Score for Epoch:  1247.17\n",
      "Ave Greedy Score for Epoch:  892.00\n",
      "\n",
      " === === === Iteration 24 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.91  Minutes\n",
      "\n",
      "Memories Elapsed:  97087\n",
      "Current Epsilon:  0.7624\n",
      "Ave Score for Epoch:  1224.17\n",
      "Ave Greedy Score for Epoch:  743.00\n",
      "\n",
      " === === === Iteration 25 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.88  Minutes\n",
      "\n",
      "Memories Elapsed:  100429\n",
      "Current Epsilon:  0.7525\n",
      "Ave Score for Epoch:  1154.83\n",
      "Ave Greedy Score for Epoch:  972.00\n",
      "\n",
      " === === === Iteration 26 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.84  Minutes\n",
      "\n",
      "Memories Elapsed:  104126\n",
      "Current Epsilon:  0.7426\n",
      "Ave Score for Epoch:  1395.17\n",
      "Ave Greedy Score for Epoch:  1062.00\n",
      "\n",
      " === === === Iteration 27 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.80  Minutes\n",
      "\n",
      "Memories Elapsed:  107958\n",
      "Current Epsilon:  0.7327\n",
      "Ave Score for Epoch:  1411.33\n",
      "Ave Greedy Score for Epoch:  1678.00\n",
      "\n",
      " === === === Iteration 28 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.78  Minutes\n",
      "\n",
      "Memories Elapsed:  111440\n",
      "Current Epsilon:  0.7228\n",
      "Ave Score for Epoch:  1244.33\n",
      "Ave Greedy Score for Epoch:  1001.00\n",
      "\n",
      " === === === Iteration 29 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.74  Minutes\n",
      "\n",
      "Memories Elapsed:  115108\n",
      "Current Epsilon:  0.7129\n",
      "Ave Score for Epoch:  1373.33\n",
      "Ave Greedy Score for Epoch:  823.00\n",
      "\n",
      " === === === Iteration 30 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.68  Minutes\n",
      "\n",
      "Memories Elapsed:  118341\n",
      "Current Epsilon:  0.7030\n",
      "Ave Score for Epoch:  1106.17\n",
      "Ave Greedy Score for Epoch:  1072.00\n",
      "\n",
      " === === === Iteration 31 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.61  Minutes\n",
      "\n",
      "Memories Elapsed:  122012\n",
      "Current Epsilon:  0.6931\n",
      "Ave Score for Epoch:  1356.50\n",
      "Ave Greedy Score for Epoch:  821.00\n",
      "\n",
      " === === === Iteration 32 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.55  Minutes\n",
      "\n",
      "Memories Elapsed:  125526\n",
      "Current Epsilon:  0.6832\n",
      "Ave Score for Epoch:  1267.00\n",
      "Ave Greedy Score for Epoch:  1763.00\n",
      "\n",
      " === === === Iteration 33 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.51  Minutes\n",
      "\n",
      "Memories Elapsed:  129248\n",
      "Current Epsilon:  0.6733\n",
      "Ave Score for Epoch:  1368.33\n",
      "Ave Greedy Score for Epoch:  1733.00\n",
      "\n",
      " === === === Iteration 34 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.50  Minutes\n",
      "\n",
      "Memories Elapsed:  132830\n",
      "Current Epsilon:  0.6634\n",
      "Ave Score for Epoch:  1289.83\n",
      "Ave Greedy Score for Epoch:  1351.00\n",
      "\n",
      " === === === Iteration 35 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.45  Minutes\n",
      "\n",
      "Memories Elapsed:  136585\n",
      "Current Epsilon:  0.6535\n",
      "Ave Score for Epoch:  1400.17\n",
      "Ave Greedy Score for Epoch:  1263.00\n",
      "\n",
      " === === === Iteration 36 of 100 === === ===\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.39  Minutes\n",
      "\n",
      "Memories Elapsed:  139847\n",
      "Current Epsilon:  0.6436\n",
      "Ave Score for Epoch:  1130.17\n",
      "Ave Greedy Score for Epoch:  1425.00\n",
      "\n",
      " === === === Iteration 37 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.35  Minutes\n",
      "\n",
      "Memories Elapsed:  143379\n",
      "Current Epsilon:  0.6337\n",
      "Ave Score for Epoch:  1240.33\n",
      "Ave Greedy Score for Epoch:  1555.00\n",
      "\n",
      " === === === Iteration 38 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.34  Minutes\n",
      "\n",
      "Memories Elapsed:  146876\n",
      "Current Epsilon:  0.6238\n",
      "Ave Score for Epoch:  1270.67\n",
      "Ave Greedy Score for Epoch:  1944.00\n",
      "\n",
      " === === === Iteration 39 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.29  Minutes\n",
      "\n",
      "Memories Elapsed:  150538\n",
      "Current Epsilon:  0.6139\n",
      "Ave Score for Epoch:  1395.17\n",
      "Ave Greedy Score for Epoch:  1311.00\n",
      "\n",
      " === === === Iteration 40 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.23  Minutes\n",
      "\n",
      "Memories Elapsed:  154167\n",
      "Current Epsilon:  0.6040\n",
      "Ave Score for Epoch:  1291.33\n",
      "Ave Greedy Score for Epoch:  2583.00\n",
      "\n",
      " === === === Iteration 41 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.17  Minutes\n",
      "\n",
      "Memories Elapsed:  157489\n",
      "Current Epsilon:  0.5941\n",
      "Ave Score for Epoch:  1151.83\n",
      "Ave Greedy Score for Epoch:  1906.00\n",
      "\n",
      " === === === Iteration 42 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.11  Minutes\n",
      "\n",
      "Memories Elapsed:  161002\n",
      "Current Epsilon:  0.5842\n",
      "Ave Score for Epoch:  1236.67\n",
      "Ave Greedy Score for Epoch:  1228.00\n",
      "\n",
      " === === === Iteration 43 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.05  Minutes\n",
      "\n",
      "Memories Elapsed:  164332\n",
      "Current Epsilon:  0.5743\n",
      "Ave Score for Epoch:  1206.00\n",
      "Ave Greedy Score for Epoch:  2565.00\n",
      "\n",
      " === === === Iteration 44 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  3.00  Minutes\n",
      "\n",
      "Memories Elapsed:  167964\n",
      "Current Epsilon:  0.5644\n",
      "Ave Score for Epoch:  1260.33\n",
      "Ave Greedy Score for Epoch:  2033.00\n",
      "\n",
      " === === === Iteration 45 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.95  Minutes\n",
      "\n",
      "Memories Elapsed:  171445\n",
      "Current Epsilon:  0.5545\n",
      "Ave Score for Epoch:  1232.00\n",
      "Ave Greedy Score for Epoch:  1967.00\n",
      "\n",
      " === === === Iteration 46 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.89  Minutes\n",
      "\n",
      "Memories Elapsed:  175030\n",
      "Current Epsilon:  0.5446\n",
      "Ave Score for Epoch:  1314.50\n",
      "Ave Greedy Score for Epoch:  2846.00\n",
      "\n",
      " === === === Iteration 47 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.85  Minutes\n",
      "\n",
      "Memories Elapsed:  179195\n",
      "Current Epsilon:  0.5347\n",
      "Ave Score for Epoch:  1610.17\n",
      "Ave Greedy Score for Epoch:  2158.00\n",
      "\n",
      " === === === Iteration 48 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.82  Minutes\n",
      "\n",
      "Memories Elapsed:  182703\n",
      "Current Epsilon:  0.5248\n",
      "Ave Score for Epoch:  1302.50\n",
      "Ave Greedy Score for Epoch:  2553.00\n",
      "\n",
      " === === === Iteration 49 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.80  Minutes\n",
      "\n",
      "Memories Elapsed:  186578\n",
      "Current Epsilon:  0.5149\n",
      "Ave Score for Epoch:  1477.67\n",
      "Ave Greedy Score for Epoch:  2688.00\n",
      "\n",
      " === === === Iteration 50 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.75  Minutes\n",
      "\n",
      "Memories Elapsed:  189998\n",
      "Current Epsilon:  0.5050\n",
      "Ave Score for Epoch:  1204.83\n",
      "Ave Greedy Score for Epoch:  3125.00\n",
      "\n",
      " === === === Iteration 51 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.73  Minutes\n",
      "\n",
      "Memories Elapsed:  193497\n",
      "Current Epsilon:  0.4951\n",
      "Ave Score for Epoch:  1234.00\n",
      "Ave Greedy Score for Epoch:  3017.00\n",
      "\n",
      " === === === Iteration 52 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.69  Minutes\n",
      "\n",
      "Memories Elapsed:  197056\n",
      "Current Epsilon:  0.4852\n",
      "Ave Score for Epoch:  1290.83\n",
      "Ave Greedy Score for Epoch:  2694.00\n",
      "\n",
      " === === === Iteration 53 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.65  Minutes\n",
      "\n",
      "Memories Elapsed:  200769\n",
      "Current Epsilon:  0.4753\n",
      "Ave Score for Epoch:  1294.67\n",
      "Ave Greedy Score for Epoch:  1506.00\n",
      "\n",
      " === === === Iteration 54 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.60  Minutes\n",
      "\n",
      "Memories Elapsed:  204137\n",
      "Current Epsilon:  0.4654\n",
      "Ave Score for Epoch:  1133.67\n",
      "Ave Greedy Score for Epoch:  1767.00\n",
      "\n",
      " === === === Iteration 55 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.54  Minutes\n",
      "\n",
      "Memories Elapsed:  207667\n",
      "Current Epsilon:  0.4555\n",
      "Ave Score for Epoch:  1278.33\n",
      "Ave Greedy Score for Epoch:  1768.00\n",
      "\n",
      " === === === Iteration 56 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.49  Minutes\n",
      "\n",
      "Memories Elapsed:  211628\n",
      "Current Epsilon:  0.4456\n",
      "Ave Score for Epoch:  1486.50\n",
      "Ave Greedy Score for Epoch:  2714.00\n",
      "\n",
      " === === === Iteration 57 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.43  Minutes\n",
      "\n",
      "Memories Elapsed:  215359\n",
      "Current Epsilon:  0.4357\n",
      "Ave Score for Epoch:  1415.33\n",
      "Ave Greedy Score for Epoch:  1818.00\n",
      "\n",
      " === === === Iteration 58 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.37  Minutes\n",
      "\n",
      "Memories Elapsed:  219126\n",
      "Current Epsilon:  0.4258\n",
      "Ave Score for Epoch:  1369.67\n",
      "Ave Greedy Score for Epoch:  2653.00\n",
      "\n",
      " === === === Iteration 59 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.31  Minutes\n",
      "\n",
      "Memories Elapsed:  222433\n",
      "Current Epsilon:  0.4159\n",
      "Ave Score for Epoch:  1143.67\n",
      "Ave Greedy Score for Epoch:  2147.00\n",
      "\n",
      " === === === Iteration 60 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.26  Minutes\n",
      "\n",
      "Memories Elapsed:  226243\n",
      "Current Epsilon:  0.4060\n",
      "Ave Score for Epoch:  1433.33\n",
      "Ave Greedy Score for Epoch:  3450.00\n",
      "\n",
      " === === === Iteration 61 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.20  Minutes\n",
      "\n",
      "Memories Elapsed:  229956\n",
      "Current Epsilon:  0.3961\n",
      "Ave Score for Epoch:  1377.67\n",
      "Ave Greedy Score for Epoch:  2229.00\n",
      "\n",
      " === === === Iteration 62 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.15  Minutes\n",
      "\n",
      "Memories Elapsed:  233787\n",
      "Current Epsilon:  0.3862\n",
      "Ave Score for Epoch:  1379.00\n",
      "Ave Greedy Score for Epoch:  1937.00\n",
      "\n",
      " === === === Iteration 63 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.10  Minutes\n",
      "\n",
      "Memories Elapsed:  237444\n",
      "Current Epsilon:  0.3763\n",
      "Ave Score for Epoch:  1232.33\n",
      "Ave Greedy Score for Epoch:  3825.00\n",
      "\n",
      " === === === Iteration 64 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  2.04  Minutes\n",
      "\n",
      "Memories Elapsed:  241110\n",
      "Current Epsilon:  0.3664\n",
      "Ave Score for Epoch:  1294.33\n",
      "Ave Greedy Score for Epoch:  2451.00\n",
      "\n",
      " === === === Iteration 65 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.99  Minutes\n",
      "\n",
      "Memories Elapsed:  245018\n",
      "Current Epsilon:  0.3565\n",
      "Ave Score for Epoch:  1503.83\n",
      "Ave Greedy Score for Epoch:  2453.00\n",
      "\n",
      " === === === Iteration 66 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.94  Minutes\n",
      "\n",
      "Memories Elapsed:  249348\n",
      "Current Epsilon:  0.3466\n",
      "Ave Score for Epoch:  1717.33\n",
      "Ave Greedy Score for Epoch:  1900.00\n",
      "\n",
      " === === === Iteration 67 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.89  Minutes\n",
      "\n",
      "Memories Elapsed:  252957\n",
      "Current Epsilon:  0.3367\n",
      "Ave Score for Epoch:  1231.17\n",
      "Ave Greedy Score for Epoch:  3089.00\n",
      "\n",
      " === === === Iteration 68 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.84  Minutes\n",
      "\n",
      "Memories Elapsed:  256547\n",
      "Current Epsilon:  0.3268\n",
      "Ave Score for Epoch:  1324.67\n",
      "Ave Greedy Score for Epoch:  2875.00\n",
      "\n",
      " === === === Iteration 69 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.79  Minutes\n",
      "\n",
      "Memories Elapsed:  260311\n",
      "Current Epsilon:  0.3169\n",
      "Ave Score for Epoch:  1377.83\n",
      "Ave Greedy Score for Epoch:  2307.00\n",
      "\n",
      " === === === Iteration 70 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.74  Minutes\n",
      "\n",
      "Memories Elapsed:  263864\n",
      "Current Epsilon:  0.3070\n",
      "Ave Score for Epoch:  1285.83\n",
      "Ave Greedy Score for Epoch:  2501.00\n",
      "\n",
      " === === === Iteration 71 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.69  Minutes\n",
      "\n",
      "Memories Elapsed:  268026\n",
      "Current Epsilon:  0.2971\n",
      "Ave Score for Epoch:  1562.83\n",
      "Ave Greedy Score for Epoch:  1726.00\n",
      "\n",
      " === === === Iteration 72 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.64  Minutes\n",
      "\n",
      "Memories Elapsed:  272716\n",
      "Current Epsilon:  0.2872\n",
      "Ave Score for Epoch:  1832.33\n",
      "Ave Greedy Score for Epoch:  2631.00\n",
      "\n",
      " === === === Iteration 73 of 100 === === ===\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.58  Minutes\n",
      "\n",
      "Memories Elapsed:  276352\n",
      "Current Epsilon:  0.2773\n",
      "Ave Score for Epoch:  1343.00\n",
      "Ave Greedy Score for Epoch:  2297.00\n",
      "\n",
      " === === === Iteration 74 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.52  Minutes\n",
      "\n",
      "Memories Elapsed:  280550\n",
      "Current Epsilon:  0.2674\n",
      "Ave Score for Epoch:  1641.00\n",
      "Ave Greedy Score for Epoch:  2923.00\n",
      "\n",
      " === === === Iteration 75 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.46  Minutes\n",
      "\n",
      "Memories Elapsed:  284707\n",
      "Current Epsilon:  0.2575\n",
      "Ave Score for Epoch:  1611.83\n",
      "Ave Greedy Score for Epoch:  2458.00\n",
      "\n",
      " === === === Iteration 76 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.41  Minutes\n",
      "\n",
      "Memories Elapsed:  288576\n",
      "Current Epsilon:  0.2476\n",
      "Ave Score for Epoch:  1366.67\n",
      "Ave Greedy Score for Epoch:  2104.00\n",
      "\n",
      " === === === Iteration 77 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.35  Minutes\n",
      "\n",
      "Memories Elapsed:  292737\n",
      "Current Epsilon:  0.2377\n",
      "Ave Score for Epoch:  1535.33\n",
      "Ave Greedy Score for Epoch:  3341.00\n",
      "\n",
      " === === === Iteration 78 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.29  Minutes\n",
      "\n",
      "Memories Elapsed:  297162\n",
      "Current Epsilon:  0.2278\n",
      "Ave Score for Epoch:  1799.50\n",
      "Ave Greedy Score for Epoch:  2647.00\n",
      "\n",
      " === === === Iteration 79 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.23  Minutes\n",
      "\n",
      "Memories Elapsed:  301416\n",
      "Current Epsilon:  0.2179\n",
      "Ave Score for Epoch:  1635.50\n",
      "Ave Greedy Score for Epoch:  3169.00\n",
      "\n",
      " === === === Iteration 80 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.17  Minutes\n",
      "\n",
      "Memories Elapsed:  305358\n",
      "Current Epsilon:  0.2080\n",
      "Ave Score for Epoch:  1452.17\n",
      "Ave Greedy Score for Epoch:  2610.00\n",
      "\n",
      " === === === Iteration 81 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.12  Minutes\n",
      "\n",
      "Memories Elapsed:  309893\n",
      "Current Epsilon:  0.1981\n",
      "Ave Score for Epoch:  1733.17\n",
      "Ave Greedy Score for Epoch:  1834.00\n",
      "\n",
      " === === === Iteration 82 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.06  Minutes\n",
      "\n",
      "Memories Elapsed:  313575\n",
      "Current Epsilon:  0.1882\n",
      "Ave Score for Epoch:  1331.83\n",
      "Ave Greedy Score for Epoch:  3843.00\n",
      "\n",
      " === === === Iteration 83 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  1.00  Minutes\n",
      "\n",
      "Memories Elapsed:  317806\n",
      "Current Epsilon:  0.1783\n",
      "Ave Score for Epoch:  1632.67\n",
      "Ave Greedy Score for Epoch:  3860.00\n",
      "\n",
      " === === === Iteration 84 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.94  Minutes\n",
      "\n",
      "Memories Elapsed:  321953\n",
      "Current Epsilon:  0.1684\n",
      "Ave Score for Epoch:  1575.83\n",
      "Ave Greedy Score for Epoch:  1246.00\n",
      "\n",
      " === === === Iteration 85 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.88  Minutes\n",
      "\n",
      "Memories Elapsed:  326058\n",
      "Current Epsilon:  0.1585\n",
      "Ave Score for Epoch:  1533.83\n",
      "Ave Greedy Score for Epoch:  3205.00\n",
      "\n",
      " === === === Iteration 86 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.83  Minutes\n",
      "\n",
      "Memories Elapsed:  330822\n",
      "Current Epsilon:  0.1486\n",
      "Ave Score for Epoch:  1936.33\n",
      "Ave Greedy Score for Epoch:  2074.00\n",
      "\n",
      " === === === Iteration 87 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.77  Minutes\n",
      "\n",
      "Memories Elapsed:  335169\n",
      "Current Epsilon:  0.1387\n",
      "Ave Score for Epoch:  1718.67\n",
      "Ave Greedy Score for Epoch:  2395.00\n",
      "\n",
      " === === === Iteration 88 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.71  Minutes\n",
      "\n",
      "Memories Elapsed:  339273\n",
      "Current Epsilon:  0.1288\n",
      "Ave Score for Epoch:  1521.83\n",
      "Ave Greedy Score for Epoch:  2474.00\n",
      "\n",
      " === === === Iteration 89 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.65  Minutes\n",
      "\n",
      "Memories Elapsed:  343768\n",
      "Current Epsilon:  0.1189\n",
      "Ave Score for Epoch:  1741.50\n",
      "Ave Greedy Score for Epoch:  1896.00\n",
      "\n",
      " === === === Iteration 90 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.59  Minutes\n",
      "\n",
      "Memories Elapsed:  348120\n",
      "Current Epsilon:  0.1090\n",
      "Ave Score for Epoch:  1715.67\n",
      "Ave Greedy Score for Epoch:  2209.00\n",
      "\n",
      " === === === Iteration 91 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.54  Minutes\n",
      "\n",
      "Memories Elapsed:  353040\n",
      "Current Epsilon:  0.0991\n",
      "Ave Score for Epoch:  1961.50\n",
      "Ave Greedy Score for Epoch:  2438.00\n",
      "\n",
      " === === === Iteration 92 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.48  Minutes\n",
      "\n",
      "Memories Elapsed:  357790\n",
      "Current Epsilon:  0.0892\n",
      "Ave Score for Epoch:  1848.67\n",
      "Ave Greedy Score for Epoch:  1744.00\n",
      "\n",
      " === === === Iteration 93 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.42  Minutes\n",
      "\n",
      "Memories Elapsed:  363023\n",
      "Current Epsilon:  0.0793\n",
      "Ave Score for Epoch:  2169.33\n",
      "Ave Greedy Score for Epoch:  2572.00\n",
      "\n",
      " === === === Iteration 94 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.36  Minutes\n",
      "\n",
      "Memories Elapsed:  368070\n",
      "Current Epsilon:  0.0694\n",
      "Ave Score for Epoch:  2112.00\n",
      "Ave Greedy Score for Epoch:  3139.00\n",
      "\n",
      " === === === Iteration 95 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.30  Minutes\n",
      "\n",
      "Memories Elapsed:  373543\n",
      "Current Epsilon:  0.0595\n",
      "Ave Score for Epoch:  2241.33\n",
      "Ave Greedy Score for Epoch:  3577.00\n",
      "\n",
      " === === === Iteration 96 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.24  Minutes\n",
      "\n",
      "Memories Elapsed:  379509\n",
      "Current Epsilon:  0.0496\n",
      "Ave Score for Epoch:  2507.67\n",
      "Ave Greedy Score for Epoch:  2109.00\n",
      "\n",
      " === === === Iteration 97 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.18  Minutes\n",
      "\n",
      "Memories Elapsed:  384685\n",
      "Current Epsilon:  0.0397\n",
      "Ave Score for Epoch:  2113.17\n",
      "Ave Greedy Score for Epoch:  2877.00\n",
      "\n",
      " === === === Iteration 98 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.12  Minutes\n",
      "\n",
      "Memories Elapsed:  390567\n",
      "Current Epsilon:  0.0298\n",
      "Ave Score for Epoch:  2532.67\n",
      "Ave Greedy Score for Epoch:  2387.00\n",
      "\n",
      " === === === Iteration 99 of 100 === === ===\n",
      "\n",
      "Training Model...\n",
      "\n",
      "Estimated time remaining:  0.06  Minutes\n",
      "\n",
      "Memories Elapsed:  396520\n",
      "Current Epsilon:  0.0199\n",
      "Ave Score for Epoch:  2595.17\n",
      "Ave Greedy Score for Epoch:  3966.00\n"
     ]
    }
   ],
   "source": [
    "startTime = time.time()\n",
    "timeLog = []\n",
    "\n",
    "printTrainInfo()\n",
    "\n",
    "for iteration in range(ITERATIONS):\n",
    "    printIter()\n",
    "    iterationStart = time.time()\n",
    "    \n",
    "    for i in range(EPISODES):\n",
    "        score,initialStateMemory,actionMemory,rewardMemory,finalStateMemory,memCount = run(initializeEnv(),agent,memCount,False)\n",
    "        gameScoreLog.append(score)\n",
    "    \n",
    "    for i in range(TESTS):\n",
    "        score = run(initializeEnv(),agent,memCount,True)\n",
    "        greedyScoreLog.append(score)\n",
    "        \n",
    "    train(initialStateMemory,actionMemory,rewardMemory,finalStateMemory,agent,EPOCHS)\n",
    "    timeLog.append(time.time() - iterationStart)\n",
    "    \n",
    "    printStats(timeLog,iteration,ITERATIONS,memCount,gameScoreLog,greedyScoreLog,EPISODES,TESTS)\n",
    "\n",
    "    if EPISODES != 25:\n",
    "        startTime = time.time()\n",
    "        timeLog = []\n",
    "        gameScoreLog = []\n",
    "        greedyScoreLog = []\n",
    "    \n",
    "    EPISODES = 25\n",
    "    TESTS = EPISODES // 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmAAAAFpCAYAAAA7jJSFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuUXnV97/HPl2QCg55muIxKJrFJ25wohshlQM6Jq0tJIXgBUtSIvUhd9OBZxaIcTzB09YSU6kqE1mjaypKCFbpUOkUaI6CRQ7QeWAdlYjSAyCEVS25CEBKrDLnxPX/sPWRmsp+ZvWff936/1po18/xmzzy/59nP5ft8f7/f92fuLgAAABTnqLI7AAAA0DYEYAAAAAUjAAMAACgYARgAAEDBCMAAAAAKRgAGAABQMAIwAACAghGAAQAAFIwADAAAoGAEYAAAAAWbWnYHxnPiiSf67Nmzy+4GAADAhDZt2vSsu/fGObbSAdjs2bM1ODhYdjcAAAAmZGb/HvdYhiABAAAKRgAGAABQsNgBmJlNMbPNZnZXeHmOmX3XzJ4ws38ys2lh+9Hh5a3h72eP+B/XhO2Pm9nirG8MAABAHSTJgH1Y0mMjLn9S0hp3nyvpeUmXhe2XSXre3X9L0prwOJnZyZIukfQGSedL+qyZTUnXfQAAgPqJFYCZ2UxJ75B0c3jZJJ0j6Y7wkFslLQl/vii8rPD3i8LjL5J0u7vvc/cnJW2VdFYWNwIAAKBO4mbAPi3pakkvhZdPkLTH3Q+Gl7dL6gt/7pO0TZLC3+8Nj3+5PeJvAAAAWmPCAMzM3inpGXffNLI54lCf4Hfj/c3I67vczAbNbHD37t0TdQ8AAKB24mTAFkq60Mx+Kul2BUOPn5bUY2bDdcRmStoZ/rxd0ixJCn8/XdJzI9sj/uZl7n6Tu/e7e39vb6xaZgAAALUyYQDm7te4+0x3n61gEv1Gd/99Sd+S9O7wsEslfTX8eX14WeHvN7q7h+2XhKsk50iaK+l7md0SAACAmkhTCf9jkm43s49L2izplrD9Fkn/aGZbFWS+LpEkd3/UzAYk/UjSQUlXuPuhFNcPAABQSxYkp6qpv7/f2YoIAADUgZltcvf+OMdSCR8AgCrbMiCtmS+t7Am+bxkou0fIQKU34wYAoNW2DEhfu1I6MBRc3rstuCxJC5aW1y+kRgYMAICquu+6w8HXsANDQTtqjQAMAICq2rs9WTtqgwAMAICqmj4zWTtqgwAMAICqWrRC6uoe3dbVHbSj1gjAAACoqgVLpQvWStNnSbLg+wVrmYDfAKyCBACgyhYsJeBqIDJgAAAABSMAAwAAKBgBGAAAQMEIwAAAAApGAAYAAFAwAjAAAKqATbdbhTIUAABkZctAsE/j3u1BtfpFK+KVkGDT7dYhAwYAQBaGg6i92yT54SAqTiaLTbeP1PCMIAEYAABZSBNEsen2aGmC2ZogAAMAIAtpgig23R6tBRlBAjAAALKQJohi0+3RWpARJAADACALaYIoNt0erQUZQVZBAgCQheFgaTKrIIf/vq0B11iLVoxeFSo1LiNIAAYAQFYIorKRNpitAQIwAABQPQ0PZpkDBgAAUDACMAAAgIIRgAEAABSMAAwAAKBgBGAAAAAFIwADAAAoGAEYAABAwQjAAAAACkYABgAAUDACMAAAgIIRgAEAABSMAAwAAKBgEwZgZnaMmX3PzH5oZo+a2V+E7V8wsyfN7Afh16lhu5nZWjPbamZbzOz0Ef/rUjN7Ivy6NL+bBQAAUF1TYxyzT9I57v5LM+uSdL+ZfT383TJ3v2PM8W+TNDf8epOkGyW9ycyOl3StpH5JLmmTma139+ezuCEAAAB1MWEGzAO/DC92hV8+zp9cJOm28O8elNRjZidJWizpXnd/Lgy67pV0frruAwBS2zIgrZkvrewJvm8ZKLtHQOPFmgNmZlPM7AeSnlEQRH03/NUnwmHGNWZ2dNjWJ2nbiD/fHrZ1agcAlGXLgPS1K6W92yR58P1rVxKEATmLFYC5+yF3P1XSTElnmdl8SddIep2kMyUdL+lj4eEW9S/GaR/FzC43s0EzG9y9e3ec7gEAJuu+66QDQ6PbDgwF7QByk2gVpLvvkfRtSee7+65wmHGfpH+QdFZ42HZJs0b82UxJO8dpH3sdN7l7v7v39/b2JukeACCpvduTtQPIRJxVkL1m1hP+3C3pdyT9OJzXJTMzSUskPRL+yXpJ7w9XQ54taa+775K0QdJ5ZnacmR0n6bywDQBQlukzk7UDyEScVZAnSbrVzKYoCNgG3P0uM9toZr0KhhZ/IOm/h8ffI+ntkrZKekHSByTJ3Z8zs7+U9FB43HXu/lx2NwUAkNiiFcGcr5HDkF3dQTuA3Jj7eAsay9Xf3++Dg4NldwMAmm3LQDDna+/2IPO1aIW0YGnZvQJqx8w2uXt/nGPjZMAAAE22YCkBF1AwtiICAAAoGAEYAABAwQjAAAAACkYABgAAUDACMAAAgIIRgAEAABSMAAwAAKBgBGAAAAAFIwAD0HxbBqQ186WVPcH3LQNl9whAy1EJH0CzbRkYvdfh3m3BZYnq7wBKQwYMQLPdd93ojaal4PJ915XTHwAQARiAptu7PVk7ABSAAAxAs02fmawdAApAAAag2RatkLq6R7d1dQftAFASAjAAzbZgqXTBWmn6LEkWfL9gLRPwAZSKVZAAmm/BUgIuAJVCBgwAAKBgBGAAAAAFIwADAAAoGAEYAABAwQjAAABoAvY8rRVWQQIAUHfseVo7ZMAAAKg79jytHQIwAADqjj1Pa4cADACagPk/7caep7VDAAYAdTc8/2fvNkl+eP4PQVh7sOdp7RCAAUDdMf8H7HlaO6yCBIC6Y/5P/rYMBAHt3u3BsN6iFdULbtjztFbIgAFA3TH/J18M8SIHBGAAUHfM/8kXQ7zIAQEYANQd83/yxRAvcsAcMABoAub/5Gf6zHD4MaIdmCQyYAAAjIchXuSAAAwAgPEwxIscTDgEaWbHSPqOpKPD4+9w92vNbI6k2yUdL+n7kv7Q3feb2dGSbpN0hqSfS3qvu/80/F/XSLpM0iFJV7r7huxvEgAAGWOIt7rqUCIkQpwM2D5J57j7GyWdKul8Mztb0iclrXH3uZKeVxBYKfz+vLv/lqQ14XEys5MlXSLpDZLOl/RZM5uS5Y0BAAAtUuMSIRMGYB74ZXixK/xySedIuiNsv1XSkvDni8LLCn+/yMwsbL/d3fe5+5OStko6K5NbAQAA2qfGJUJizQEzsylm9gNJz0i6V9K/Sdrj7gfDQ7ZL6gt/7pO0TZLC3++VdMLI9oi/GXldl5vZoJkN7t69O/ktAgAA7VDjEiGxAjB3P+Tup0qaqSBr9fqow8Lv1uF3ndrHXtdN7t7v7v29vb1xugcAANqoxrtAJFoF6e57JH1b0tmSesxseBL/TEk7w5+3S5olSeHvp0t6bmR7xN8AAAAkU+MSIRMGYGbWa2Y94c/dkn5H0mOSviXp3eFhl0r6avjz+vCywt9vdHcP2y8xs6PDFZRzJX0vqxsCAABapsYlQuJUwj9J0q3hisWjJA24+11m9iNJt5vZxyVtlnRLePwtkv7RzLYqyHxdIknu/qiZDUj6kaSDkq5w90PZ3hwAANAqNS0RYkFyqpr6+/t9cHCw7G4AAABMyMw2uXt/nGOphA8AAFAwAjAA7bRlQFozX1rZE3yvQeFGAM0RZw4YADTLcPXs4QKOw9WzpVrOJQFQP2TAALRP2dWzyb4BrUcGDED7lFk9m+wbMFpNN9NOiwwYgPYps3p22dk3oEpqvJl2WgRgANqnzOrZNd67Dshciz+QEIABaJ8yq2fXeO86IHMt/kDCHDAA7VRW9exFK0bPAZNqs3cdkLnpM8Phx4j2hiMDBgBFqvHedUDmaryZdlpkwACgaDXduy6Rlq5sQ0LDj4kWPlYIwAAA2aLUBpJowweSCAxBAgCy1Wll29c/RgFaIEQGDADGw1Bacp1WsA09F3xJZMXQemTAAKCTFheJTCXuCraW1HsCohCAAUAnLS4SmUrUyrZOWlDvCYhCAAYAnbS4SGQqUaU2uo+PPrYF9Z5qsfl6HfrYMMwBA4BOWlwkMrWxK9vGroyU2lHvqQ4rQuvQxwYiAwYAw8ZmAeae19oikZlrawHaOgxj16GPSdUgo0cGDACk6CzAD78kvfH3pCe+ySrILLSx3lMdhrHr0MckapLRIwADAKlzFuCJb0pXPVJOn1B/dRjGrkMfkxgvo1ehAIwhSACQmpcFQDXUYa/DOvQxiZo8lwnAAEDq/Gm/rlkAVEMd5r7VoY9J1OS5zBAkgGaZbOX6RSvauUoP+avD3Lc69DGumjyXyYABaI40leublgUAilSlVYc1eS6bu5fdh476+/t9cHCw7G4AqIs18ztMJp7FRHogL51qvFUw6MmbmW1y9/44x5IBA9AcNZl8i4aoUtanTE2sI1YAAjAAzVGTybdoADZqP4wPPpNCAAagOZq2nB7VRdbnMD74TAoBGIDmqMnkWzQAWZ/D+OAzKZShANAsTVpOj+pqWvX4NIafb5Mp/9JiBGAAACRVk1pTheGDT2IMQQIAkBTD3UiJDBgAIL7J7jRQtjz6TdancOs279ANGx7Xzj1DmtHTrWWL52nJaX1ld2tSJsyAmdksM/uWmT1mZo+a2YfD9pVmtsPMfhB+vX3E31xjZlvN7HEzWzyi/fywbauZLc/nJgEAclHX0gt17TdGWbd5h66582Ht2DMkl7Rjz5CuufNhrdu8o+yuTUqcIciDkj7q7q+XdLakK8zs5PB3a9z91PDrHkkKf3eJpDdIOl/SZ81siplNkfR3kt4m6WRJ7xvxfwAAVVfX0gt17XcTpShee8OGxzV04NCotqEDh3TDhsePOHbd5h1auHqj5iy/WwtXb6xkkDbhEKS775K0K/z5P8zsMUnj5fsuknS7u++T9KSZbZV0Vvi7re7+E0kys9vDY3+Uov8A0FxVG+6ra+mFuva7acZuWTSciZRiPa537hmK1T6cKRsO1oYzZZIqNVyZaBK+mc2WdJqk74ZNHzKzLWb2eTM7LmzrkzRybe72sK1TO4A2SLNtSxu3fKnisFldC27Wtd9NkzITOaOnO1Z7kkxZmWIHYGb2SklfkfQRd/+FpBsl/aakUxVkyP56+NCIP/dx2sdez+VmNmhmg7t3747bPQBVliaYqGIgUoQqDpvVteBmXfvdNCkzkcsWz1N315RRbd1dU7Rs8bxRbXEzZWWLFYCZWZeC4OuL7n6nJLn70+5+yN1fkvT3OjzMuF3SrBF/PlPSznHaR3H3m9y93937e3t7k94eAFWUJpioYiBShCoOm9W19EJd+900KTORS07r06qLT1FfT7dMUl9Pt1ZdfMoRw4pxM2Vlm3AOmJmZpFskPebunxrRflI4P0ySflfSI+HP6yV9ycw+JWmGpLmSvqcgAzbXzOZI2qFgov7vZXVDAFRYmmCiioFIEapaab2upRfq2u8myaB47ZLT+iacx7Vs8bxRc8Ck6ExZ2eJkwBZK+kNJ54wpOXG9mT1sZlskvVXSVZLk7o9KGlAwuf4bkq4IM2UHJX1I0gZJj0kaCI8F0HRpPvm2df4Ow2bxtHF+YF0VlImMmykrm7kfMQ2rMvr7+31wcLDsbgBIa+zqJykIJuK8+Kb527qr2irIqmnzYwOVZGab3L0/zrFUwgcQT5pgIM1mvW3e6Jdhs/GNNz+Q+w0VRwAGYGIp6/e8fNxk3xQJRBClrfMD0Qhsxg1gYm1diYhqa+v8QDQCARiAiZFpQBWxUAE1RgAGYGJkGlBFaVfVsYISJWIOGICJZVC/B8jFZOcHZjGvEUiBDBiAiVFJHE3DvEaUjAwYgHiquBKROlnJ1eE+K6KPzGtEyQjAANQTQ0jJ1eE+K6qPVd3qCa3BECSAemIIKbk63GdF9ZEVlCgZARiAemIIKbkk91lZKwSLOq/Ma0TJGIIEUE8MISUX9z4rc6iyyPNaxXmNaA0yYADqiSGk5OLeZ2UOVXJe0RIEYADqiSGk5OLeZ2UO73Je0RLm7mX3oaP+/n4fHBwsuxsA0C5r5ncYBpwlXfVI8f0BasLMNrl7f5xjyYABAEZjGBAVsG7zDi1cvVFzlt+thas3at3mHWV3KVNMwgcAjDY83Ff1gq1orHWbd+iaOx/W0IFDkqQde4Z0zZ0PS5KWnNZXZtcyQwAGADgSKwRRohs2PP5y8DVs6MAh3bDh8cYEYAxBAgCapawaZnVRg/tn556hRO11RAAGAGiO4Rpme7dJ8sM1zCoYZJQi6f1TUrA2o6c7UXsdEYA1UQ0+3aBCeLzUD+esszpst1SmJPdPicHsssXz1N01ZVRbd9cULVs8L/frLgoBWNPw6Q9JlP14IZBIruxzVnVsUTW+JPdPicHsktP6tOriU9TX0y2T1NfTrVUXn9KY+V8Sk/CbZ7wnDBNqMVaZj5cyt7ups6Y9x7cMZLvaki2qxpfk/ik5mF1yWl+jAq6xyIA1DZ/+kESZjxeGiianSc/xPLJ51DAbX5L7p1PQmiKYbXptryQIwPJS1tBKDk8YNFiZj5cmBRJZiPua0aTneB5BOFsZjS/J/ZNxMDtc22vHniG5Dtf2amsQxhBkHsocWlm0YvR1S3z6Q2dlPl4YKjosyWtGk57jeQXh1DAbX9z7p1NBXincrirZsHEbanslQQYsD2UOrfDpD0mU+XhhqOiwJK8ZTXqONymb11QLlgb7f67cc3gf0EkOG7ehtlcSZMDyUPbQCp/+kERZjxe2uzks6WtGU57jTcrmtUWKRSAzerq1IyLYSlvba93mHbphw+PauWdIM3q6tWzxvFpk1AjA8sDQChBPUwKJtNr6mkEQXj8pEgzLFs8btb+jlL62V533jGQIMg8MrQBIos2vGWOHuAi+qi3FsHEetb3Gm1dWdWTA8sCnOgBJ8JqBukg5bJx1ba86zysjAMsLQysAksjjNSPrIqdADh8W0szhymteWREIwACgidhpAHnJ8MNC2jlcecwrKwpzwACgidhpADWQdg5XnfeMJAMGAE1UdjkcIIYs5nDVdc/ICTNgZjbLzL5lZo+Z2aNm9uGw/Xgzu9fMngi/Hxe2m5mtNbOtZrbFzE4f8b8uDY9/wswuze9mAUDLUeQUNdBprlYd5nClFWcI8qCkj7r76yWdLekKMztZ0nJJ97n7XEn3hZcl6W2S5oZfl0u6UQoCNknXSnqTpLMkXTsctLVeWftGojp4DCBrbS5tkQbPxUItWzxP3V1TRrWNO4erQednwiFId98laVf483+Y2WOS+iRdJOkt4WG3Svq2pI+F7be5u0t60Mx6zOyk8Nh73f05STKzeyWdL+nLGd6e+mGiLHgMIA9NLG2R96pOnouFGx46jLUKsmHnx4I4KebBZrMlfUfSfElPuXvPiN897+7Hmdldkla7+/1h+30KArO3SDrG3T8etv8vSUPu/ledrq+/v98HBweT3qZ6WTO/QwXsWYf33UKz8RgAJjb2zVcKMnpZ7oPJc7HaanB+zGyTu/fHOTb2Kkgze6Wkr0j6iLv/YrxDI9p8nPax13O5mQ2a2eDu3bvjdq9caVKiTJQFjwFgYkWs6uS5WG0NOz+xAjAz61IQfH3R3e8Mm58OhxYVfn8mbN8uadaIP58paec47aO4+03u3u/u/b29vUluSzmGP5VNYmd4SUyUBY+BpmvQnJVSFfHm28TnYpMefw07P3FWQZqkWyQ95u6fGvGr9ZKGVzJeKumrI9rfH66GPFvS3nAe2QZJ55nZceHk+/PCtnpL+6mMibLgMdBcaT+g4bAi3nyb9lws8PG3bvMOLVy9UXOW362Fqzdq3eYdmV9H085PnAzYQkl/KOkcM/tB+PV2SaslnWtmT0g6N7wsSfdI+omkrZL+XtKfSFI4+f4vJT0Ufl03PCG/1tJ+KluwNJjDMH2WJAu+ZzmnAdXHY6C5KIaanSLefJv2XCzo8TdczX7HniG5Dlezf2j957LNvjXs/CSahF+0WkzCr8GkQAAlWdmjiKmukkxauafo3tQfe1smU9Djb+HqjUfsx3jhUffrk9NuUbf2HW7MetFEBSWZhE8l/LRS7gwPoMGmz+zwAa2ec1ZKl8eG5Q32QvdrdOzQruj2DK8nqmr91VMHRgdf0uHsG+dQEntBptewlGihmjQ5FIjSsDkrqJfrD7xXL/i0UW0v+DRdf+C9mV5PVNX6GfZs9ME1XbGYBzJgWeBTWXINK6gHRGpiMVTUxq2/PEvPHbVfV08d0Az7uXb6Cbr+4FJ9bd9ZWpnh9SxbPE/X3PnwqE21d+lE9SkiCCP7+zICMJRjvMmhvDmhSfiAlqt1m3fEq6LeQjN6urV+z5u1fv+bR7X3ZbzPYlQ1+50nX62+h69les44CMBQjoYV1ANQvOHVd8OZl+HVd5IIwhSdmRp3n8UUlpzWN+Y+P0eafRzZ33EwB6yqmj4/qmEF9YBWK+n16oYNj48KLiRp6MAh3bDh8UKuv+qWnNanVRefor6ebpmCzNeqi08pLDhdd2ihFu5bqzkvflEL963VukMLC7neuiADVkVtmB/F6lGgGUp8vYpafTdeexsdmZkqBtnJiZEBq6I2FG9k9SjQDCW+XkWtvhuvHcUhOzkxMmBV1Jb5UUxOBuqvxNerIuc4IRmykxMjA1ZFzI8CUBclvl6VPccJnZGdnBgZsCpifhSAuij59aqsOU4YH9nJiRGAVRHFGwHUBa9XiBBVG4wabaOxGTcAAAWjgGwzsRk3AAAVRYkGSARgAKpmywDDWaiFyWaxxivRULUAjExdfgjAAFRHG4oQoxHSZLHqUqKBTF2+KEMBoDraUIQYjZCm0GhdSjRQTDVfZMAAVEdbihCj9tJksZYtnqf7/+Wz+ohu1wx7Vjv9RH1al+jNi/8k626mUpdMXV2RAQNQHRQhRk2kyWItmfKAVnfdrJlHPaujTJp51LNa3XWzlkx5IOtuplKXTF1dEYABqI5FK4IiniNRhLjy1m3eoYWrN2rO8ru1cPVGrdu8o+wu5W7Z4nnq7poyqi12odH7rtPUQy+Oapp66MXKDbV3uo1vfV1v6853HhiCBFAdFPWsnbZO1E5VaLQmQ+1Rt/Gtr+vVVzbtaN35zgOFWAEgKy0sobFw9UbtiJgT1NfTrQeWn1NCj2pgzfxghe9Y02dJVz2S+9WnKS3B+R5fkkKsDEECQBaGS2js3SbJD5fQ2DJQds9yxUTtSShxqH04Y7ljz5BchzNYcYcROd/ZIQADgCy0tIQGE7UnYcFS6YK1QcZLFny/YG0h2dK0pSU439lhDhgAZKEm83qytmzxvFFzwKQEk9ELVFRF99jXs2BpKcPTaTNYdTnfdUAABgBZmD6zw7yeZpfQSDUZvSBFLRSow4KEGT3dkXO44maw6nC+64JJ+ADSaeHE80hjt1GSgnk9BQ0tobOiJo7XYYL62CBRCjJYqy4+hSAqA0km4ZMBAzB57N14GCU0KquoieN5XU+Ww6dksKqDAKxOyDSgasabeF7AY7OoeT2xlTSvpy0me747DbtN7+7SwtUbM3v8pB3ei5LHsOaS0/oIuCqAVZB1UeQS9y0DQZ2alT3B94Yvo6+sOpyHEieep11Oj3pJc76jKrp3HWX61f6DmT5+UlXH74ANsZuLAKwuilri3tJaRpXT6Tzc9T+qFZSVuHcjb0ztkuZ8LzmtT6suPkV9Pd0yBXOyXnnMVB04NHoO9Hj/L852S1HXk3ZuVafhyx17htgOqOYYgqyLojINJQ8pIdTpPAx+XlL4plGF+VaLVkRPPC+goCQFIdsl7fkeO+w2Z/ndsf9fkmHArIf3Og1rWtiPifoTV+WG81uADFhdFJVpaGkto8rpeH+PWbVcdqHPEgtKUhCyXbI+30n+X5nZ1qhhTdMRrwSp+sNwfjkIwOqiqK0rShxSwghJ7u+yg+MFS4P961buCb4XlI3LY74Nqivr853k/5WZbY0a1uxUPGqyw5IM55eDIci6KGqJe4lDSpVU1srTqPMQ+blXrQ2OWU7fLlmf7yT/L4/VjUmMHdbsVG9sssOSDOeXY8JCrGb2eUnvlPSMu88P21ZK+m+SdoeH/Zm73xP+7hpJl0k6JOlKd98Qtp8v6TOSpki62d1XT9Q5CrGWhHIXgbILa449D3PPk374JQp9AgWrWvHSqP50+HgWqwhsHQrI1kWSQqxxArDflvRLSbeNCcB+6e5/NebYkyV9WdJZkmZI+t+S/nP46/8n6VxJ2yU9JOl97v6j8a6bACxjBFbJrJnfYWuZWcFQWxk4h0ApqjZJfWx/ogIoKQjMnlz9jgn/V5UCzDrLtBK+u3/HzGbHvO6LJN3u7vskPWlmWxUEY5K01d1/Enbw9vDYcQMwZIiK5clVcUEChT4boWpv5knUue9pVK14adxhyTjDpAznlyPNHLAPmdn7JQ1K+qi7Py+pT9KDI47ZHrZJ0rYx7W9Kcd1IivISybV0c2Xkqw4bNndS57433bLF8yKzWHEXKVQtwGyDya6CvFHSb0o6VdIuSX8dtlvEsT5O+xHM7HIzGzSzwd27d0cdgsmoYjan6opaeYrcxSmiWZQ6rzirc9+bLo8isMjXpDJg7v708M9m9veS7govbpc0a8ShMyXtDH/u1D72f98k6SYpmAM2mf4hAtmc5NhcuRFyy9pMcj5enVec1bnvbUAWq14mlQEzs5NGXPxdScMzktdLusTMjjazOZLmSvqegkn3c81sjplNk3RJeCyKQjZnUtYdWqiF+9Zqzotf1MJ9a7Xu0MKyu4SEcsnapNiyq84FZOvc97jyyJZWKQOL6pgwADOzL0v6v5Lmmdl2M7tM0vVm9rCZbZH0VklXSZK7PyppQMHk+m9IusLdD7n7QUkfkrRB0mOSBsJjUZQSK5bXFdWhmyGXrE2KvVnLLiCbJhgou+95y+M5z+sIOomzCvJ9Ec23jHP8JyR9IqL9Hkn3JOpdE5VZRoAVdImMlzkhzV8fuRTRTDGnstOKMylYyTbZVWhxViemHY5t+mq5PJ7zvI6gEyrhZyD2smxKQdQK812aIe3qsEgp51SOnauTNjCK+/dZBANNnmeUx3NIA/IlAAAWDUlEQVSe1xF0wl6QKSVKL6cYtkD+xg7N9BzbFXlcFee7MMeks1xWh2U8pzLtPLW4f08wML485rjlNm9uy0BQLHplT/A9xvxDVAsZsJQSfaKkFERlRWUQuo4ydU0xHTh0eDFuFee7dMp+9G27S2f+29+wglM5ZG0yXiGbNjCK+/dl72lYdXlkS3PJwDKa0ggEYCkleuFsQSmItFWyo/5eyn/OSVQgfeAlV093l15x9NRS5rvEvS+j+n7uoX/V/O/fImlf0DD8Av3Ug9IT35w4aGDLo4llOKeyU2A0vbsr1rywuIFVLsFASlWqrJ/HHLdc5s1RWLsRJtwLskx12Asy0SamZW/unLO0+4lF/X3XUSaZjshCZV1gcM7yuyMrA8fZRy0PSe7LqL7fP+1KzTzq2Yj/PGbL3qjH35YBHfzqn2rqoRdfbjo45RhNvehvGvE4zUqWgUPax36nx8u7zujTt368u/APNHGxB+EkrexRdC1zk1buKbo3GCHJXpDMAUsp0bLsBKUgoub0VH2eTx7zWA685KPegJL+z7jSztPI+twkuS+j+jjDooIv6YgX7Yg5iC98fcWo4EuSph56US98nZpxw7IuLRA1T+2Vx0yN/diP+vt3ndGnr2zacUQfJemB5efoydXv0APLzyk10KGy/iR1GjVp0GhKGzAEmVLi9HKMYYuoOT3L/vmHoz4NV3EPtrzmsaQ9No40QzN5VFpPcl9G9X2XTlSfOgVhY4yZg3jM0M8iDztm6GepyiREqdLwUxJ5lBYYO09tzvK7I4/r9NiI2pw5TR+LODcsCpikRSuiR1MorF0rBGAZyHqCb6dM0FhDBw5p5fpHC3kDi/NinHaCb6e/73RsltLM00j6Zpz1fRnV950nX62+h68dM09kzPDjsDGfmne+dELk8OXOl054uU9ZBJl13ti5iMAh7fMpTR+LOjcsCpgktklrBAKwCkryIr5n6ID2DB2QlN+LZNwX47QTfKP+vtM8mDwmDU82kE7yRpfXfXlk38+RZh83+gV67nnSD7804afmm6f9ga4+8Fkda/tfbnvBp+n6g6Nf3IssUFm1TFkRgUPa51OSPo69f1/Yf7CQ4qF5LQqo2uMlFxTWrj3mgFVQmhfxPOZPxJ2nkbbeUtTf3/CeN+qGd78x2xpOGUsyfyztfSkp/lyzBUulqx4JJuVe9Yj0zk/FmoN46jsu1wq/XNtfOlEvuWn7Sydq+YE/1vqX3nzEVRRRoLKKW7kUsSVP2udT3D5G3b/Pv3Ag8n9mPTSYR422Kj5e4qr6PF9kiwxYBcXNBHWS9YtkkgxP2uHYTn9fpYBrrCSf4tPcl5kMC8X41Bz8rz/RezcsGpURUcSbctoClXEyNFXcyqWoLXnSPJ/i9jHq/u0kbYavU2Yq7ykcZT9e4qjzkDwmhwCsgsbbK27sMEHUJ9Ws508wT2N8Sd6M09yXRb6xTBT8ScUVqKzqRO24gUOZw2Fx+hj3fkx6vsfe7re+rldf2bQj9wCjqo+XidQ1cMTkEYAVLO6LcZxMUB5vilGymKdRuTkZGRcajftmnOa+LPONpcwClWV/AEjz2K1DVqPT/ZumCHHU7f7ig08dsQQkjwCj7MfLZNU1cMTkEYAVKOsX47yGQaLecFZdfEpz3oRK3MZjvOzmRCUeyn5jyWMT5jj/M2nQmmeB1KSP3TpkNTrdvysvfEOmCyw6TZ7otFhlsuewitX+4yj7+Y3iUQl/PBlnSRJVzS/IRMMEUvrK1JW73Wvmd9gSalYwWb1gcauBj3vclAdiPVaL2uop64xn3P+XdWX1tI/dqu2w0EnW56vT7Y4y9r7M4hxWLuMeA7sCNEOSSvhkwDrJIUtStRRzUcMEZd/usS/G97+4XRZ1YMxN0bN+cY+bJemY8ZzyQKzHalEFftNmjdJM1E6bcRp73Z3q0sV97NYlq5F1drPT7R5biS4qM5VF1jCPbG3eilrYgeogAOskh81Oq/ZinHaYIK6ibnen7M7YYGDn0SeoL2qrnhjbeJRd9T7yjWVNvMdqkgK/aet7nXvoX3X1tAHNsGe100/U9QeX6oYN0yb8n2nv36yLj3YoXRv7sVvX4bC0Ot3uqL0px57Xsj+wlamOgSMmjwCsk07ZkL3bwiGs5MOSZb8Yx/10HyXqDeeh9Z/TrO/foFf5bj1jvdp2+jKdeeEHjziuiNvd6Y37mK6jjgg6PnlgqT457RZ1a9/hxpjbeOQxpyd1gNrxsTq6vaitnvp/ca9Wdd38ciHXmfasVnfdrGt+IUnjD9ulvX+zLj7qipe16aStWY00t7tqH1SBvBCAdTJ9ZvQ8Idnh9oTDkkW9GMfNBHX6dB/nDeeh9Z/T/E1/rm7bL5n0Gu3W9E1/roekI4KwIm53pzfuqPpG6196s2y/9JneryUOpPP4dJ46QO30WB2T0Stqq6drpv2zjtX+UW3H2n5dM+2fJa0a92/T3r9x78uogL0TVzBPKc5jt4g6V3Ux2dtd9gdVoCgEYKGxL5yfPvlPdWacvfQSDkvm/WKcJBPkki466n4tm3p4qOjTukTHnH7JhMMEs75/QxB8jdBt+zXr+zdIEVmwvG930gBo8NfOla4aPxiIksen89QBasyNeYva6unVHTYB79Q+Utr7N4/io3En3FdutW9NtTVriPYhAFP0C+f7H/p13XbmX+jMf/ubl7Mkvndb5ORt39thUncJkmSCLjzq/iOHiqbcrKm/8UZpyfgB5at8t6Ju9Kt84jfZPIxXy2jfwZcy+zSd16fzVAFqzI154xb4TftmZx0yckPdr9G5E5TayOL+Lav4aB1KTtRFW7OGaBcCMHV+4fzIj+bqgeWHyxI8vfK39BrtPuLvn9aJek3uvYwnSSbo6qkDozZclqSph16MldF7xnoj74tnrJz7YrxaRtLk6m5Fqeyn85gb88Yp8JtUnOzxwSnHaMWv3qUd+4O2Ttmhou7fPIqPtnnyOIDkCMAU/4Vz1f73jMoYSdILPk2rDrxHn8m1h/ElyQTNsJ9H/5MY5Ri2nb5M04fngIWGfJq2nbGslABsojfu8eppJR0q4tP5YXGzxx//1bt0x/6zRv1tp+xQEfdvHsVHi5w8Xsc6VwBGIwBT/BfOwV87V8t/EWSOZtjPtdNP0PUHl2rTr51bVFcnlCQT9KK9RscO7Tryn8Qox3DmhR/UQ1K4CvJZPWMnatsZ0asgO8n6TaSoWlE4LG72+Nbld0f+fVnZoTwybUVNHmeuGdAMBGCK/8IZHLdf6/e/edRxqyq0OidJJkhbros1ebuTMy/84MsT7l8TfsVV5psIQ0XZiXtfVrG0QNaZtqKGT/kAATQDAZjiv3BWcv5PxHZJS05bGq9PMSdv56HMN5Eig4G4tdLyUMQwVdz7si2lBYoYPuUDBNAMBGChuC+cebzATvqNMovtkmJO3k5tTKDY/4sLtENvPuKwIt5EigoGktRKy1pRGca492UlP7zUVBWziQCSIwArWao3yhy2S8pFRKC4etot8v1BUdSRingTKSoYSForLUtFZRiT3JcsXshGW7KJQNMRgI2jiCGcVG+UMbegKV1EoNitffpY14DW7xs9n66oN5FUwUDEsG9UwFtmrbQih6kIrIpFNhFoBgKwDooawkn1RhlzC5rSdQgIZ9jPY2/xUog4gVWCYd8ya6UxTNVsBL1A/R1VdgeqarzMVJY6vSHGeqNctCJYtThSglWMhekQENr0mXpg+Tl6cvU79MDyc4I3lC0DwWbnK3uC71sGiunjcGC1d5skPxxYjb3+8YZ9x9h2+jIN+bRRbUM+TdtOX5Zx54+0bPE8dXdNGdXGMBUAVAcBWAdFDeGkeqNcsFS6YK00fZYkC75fsLZa87+k+IFi3CBIQYZy4eqNmrP8bi1cvVHrNu9I18e4gVWCYd8zL/ygHjnj4/qZevWSm36mXj1yxscLWQW55LQ+rbr4FPX1dMsU7Ge46uJTyJoAQEUwBNlBUUM4qedzFLWKMY245S5iLirIZXg4bmCVcNg3Ta20tBimAoDqIgDroMiVRq14o4wTKMYMgnJZ4Rc3sFq0IlXxWgAAJIYgO2IIpwSdFg+Mac9leDjuMGldhn0BAJU2YQbMzD4v6Z2SnnH3+WHb8ZL+SdJsST+VtNTdnzczk/QZSW+X9IKkP3L374d/c6mkPw//7cfd/dZsb0r2WpGZqpKY2aVchoeT7ApQh2FfAEClxRmC/IKkv5V024i25ZLuc/fVZrY8vPwxSW+TNDf8epOkGyW9KQzYrpXUL8klbTKz9e7+fFY3pGqKqCHWODGDoNyGhwmsAAAFmTAAc/fvmNnsMc0XSXpL+POtkr6tIAC7SNJt7u6SHjSzHjM7KTz2Xnd/TpLM7F5J50v6cupbUEFlbjRdezGCIApRAgDqbrKT8F/t7rskyd13mdmrwvY+SSNnMm8P2zq1V0fM6uZxlLnRdG7i3j8Z3o/jYXgYAFBnWa+CjNh4RT5O+5H/wOxySZdL0mtf+9rsejaeLDa1HqHIbWAKEff+yfh+BACgqSa7CvLpcGhR4fdnwvbtkmaNOG6mpJ3jtB/B3W9y93537+/t7Z1k9xJKUN08jlTV7aso7v2T8f0IAEBTTTYAWy/p0vDnSyV9dUT7+y1wtqS94VDlBknnmdlxZnacpPPCtmrIeFPrxm0DE/f+qcvm4HkoawslAEAtxSlD8WUFk+hPNLPtClYzrpY0YGaXSXpK0nvCw+9RUIJiq4IyFB+QJHd/zsz+UtJD4XHXDU/Ir4SMN7Vu3CTxuPdPXTYHzxpDrwCAhCxYsFhN/f39Pjg4mP8VjX0DlYL6UxTYDMS9f9p6P66Z3yHwnCVd9Ujx/QEAlMLMNrl7f5xjqYQvUd18InHvn7bej20eegUATAoZMCAtMmAAAJEBA4oVdx9JAABCBGBAWkmHXlkxCQCtl3UhVqCd4u4jyYpJAIDIgAHFolgtAEAEYGiSTkN7VRryY8UkAEAMQaIpOg3tPfWg9MMvVWfIr63FagEAo5ABQzN0Gtrb9IVqDfmxYhIAIAIwNEWnITw/lOz4vLW1WC0AYBSGINEMnYb2bEp0EFbmkF/cFZMAgMYiA4Zm6DS0d8YfMeQHAKgcAjA0Q6ehvXd+iiE/AEDlsBckAABABtgLEgAAoMIIwAAAAApGAAYAAFAwAjAAAICCEYABAAAUjAAMAACgYARgAAAABSMAAwAAKBgBGAAAQMEIwAAAAApGAAYAAFAwAjAAAICCEYCh+rYMSGvmSyt7gu9bBsruEQAAqUwtuwPAuLYMSF+7UjowFFzeuy24LEkLlpbXLwAAUiADhmq777rDwdewA0NBOwAANUUAhmrbuz1ZOwAANUAAhmqbPjNZOwAANUAAhmpbtELq6h7d1tUtzT2PifkAgNpiEj6qbXii/X3XBcOO02cGwdcPv8TEfABAbRGAofoWLB0dWK2Z33liPgEYAKAGGIJE/TAxHwBQc6kCMDP7qZk9bGY/MLPBsO14M7vXzJ4Ivx8XtpuZrTWzrWa2xcxOz+IGoIWYmA8AqLksMmBvdfdT3b0/vLxc0n3uPlfSfeFlSXqbpLnh1+WSbszgutFGnSbmL1pRTn8AAEgojyHIiyTdGv58q6QlI9pv88CDknrM7KQcrh9Nt2CpdMFaafosSRZ8v2At878AALWRdhK+S/qmmbmkz7n7TZJe7e67JMndd5nZq8Jj+yRtG/G328O2XSn7gDYaOzEfAIAaSRuALXT3nWGQda+Z/XicYy2izY84yOxyBUOUeu1rX5uyewAAANWTagjS3XeG35+R9C+SzpL09PDQYvj9mfDw7ZJmjfjzmZJ2RvzPm9y93937e3t703QPAACgkiYdgJnZK8zsPw3/LOk8SY9IWi/p0vCwSyV9Nfx5vaT3h6shz5a0d3ioEgAAoE3SDEG+WtK/mNnw//mSu3/DzB6SNGBml0l6StJ7wuPvkfR2SVslvSDpAymuGwAAoLYmHYC5+08kvTGi/eeSFkW0u6QrJnt9AAAATUElfAAAgIIRgAEAABSMAAwAAKBgBGAAAAAFIwADAAAoGAEYAABAwSyoDlFNZrZb0r8XfLUnSnq24OtEPJybauP8VBfnpro4N9WW9Pz8urvH2san0gFYGcxs0N37y+4HjsS5qTbOT3VxbqqLc1NteZ4fhiABAAAKRgAGAABQMAKwI91UdgfQEeem2jg/1cW5qS7OTbXldn6YAwYAAFAwMmAAAAAFIwALmdn5Zva4mW01s+Vl96ftzGyWmX3LzB4zs0fN7MNh+/Fmdq+ZPRF+P67svraVmU0xs81mdld4eY6ZfTc8N/9kZtPK7mMbmVmPmd1hZj8Onz//hedNdZjZVeFr2iNm9mUzO4bnTjnM7PNm9oyZPTKiLfK5YoG1YYywxcxOT3v9BGAK3kgk/Z2kt0k6WdL7zOzkcnvVegclfdTdXy/pbElXhOdkuaT73H2upPvCyyjHhyU9NuLyJyWtCc/N85IuK6VX+Iykb7j76yS9UcE54nlTAWbWJ+lKSf3uPl/SFEmXiOdOWb4g6fwxbZ2eK2+TNDf8ulzSjWmvnAAscJakre7+E3ffL+l2SReV3KdWc/dd7v798Of/UPAm0qfgvNwaHnarpCXl9LDdzGympHdIujm8bJLOkXRHeAjnpgRm9muSflvSLZLk7vvdfY943lTJVEndZjZV0rGSdonnTinc/TuSnhvT3Om5cpGk2zzwoKQeMzspzfUTgAX6JG0bcXl72IYKMLPZkk6T9F1Jr3b3XVIQpEl6VXk9a7VPS7pa0kvh5RMk7XH3g+FlnkPl+A1JuyX9Qzg8fLOZvUI8byrB3XdI+itJTykIvPZK2iSeO1XS6bmSeZxAABawiDaWh1aAmb1S0lckfcTdf1F2fyCZ2TslPePum0Y2RxzKc6h4UyWdLulGdz9N0q/EcGNlhPOJLpI0R9IMSa9QMLQ1Fs+d6sn8NY4ALLBd0qwRl2dK2llSXxAysy4FwdcX3f3OsPnp4bRv+P2ZsvrXYgslXWhmP1UwXH+OgoxYTzisIvEcKst2Sdvd/bvh5TsUBGQ8b6rhdyQ96e673f2ApDsl/Vfx3KmSTs+VzOMEArDAQ5LmhitRpimYFLm+5D61Wjin6BZJj7n7p0b8ar2kS8OfL5X01aL71nbufo27z3T32QqeKxvd/fclfUvSu8PDODclcPefSdpmZvPCpkWSfiSeN1XxlKSzzezY8DVu+Pzw3KmOTs+V9ZLeH66GPFvS3uGhysmiEGvIzN6u4FP8FEmfd/dPlNylVjOzN0v6P5Ie1uF5Rn+mYB7YgKTXKngxe4+7j51EiYKY2Vsk/U93f6eZ/YaCjNjxkjZL+gN331dm/9rIzE5VsDhimqSfSPqAgg/bPG8qwMz+QtJ7Faz03izpjxXMJeK5UzAz+7Kkt0g6UdLTkq6VtE4Rz5UwYP5bBasmX5D0AXcfTHX9BGAAAADFYggSAACgYARgAAAABSMAAwAAKBgBGAAAQMEIwAAAAApGAAYAAFAwAjAAAICCEYABAAAU7P8D7TMJERRSsYgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def getMovingAve(movingAveSize, scores):\n",
    "    movingAve = []\n",
    "    \n",
    "    for i in range(len(scores) // movingAveSize):\n",
    "        if (i + 1) * movingAveSize < len(scores):\n",
    "            movingAve.append(np.mean(scores[i*movingAveSize:(i+1)*movingAveSize]))\n",
    "    return movingAve\n",
    "\n",
    "\n",
    "def mapScores():\n",
    "    movingAveSize = 5\n",
    "    \n",
    "    movingAveA = getMovingAve(movingAveSize*5, gameScoreLog)\n",
    "    movingAveB = getMovingAve(movingAveSize, greedyScoreLog)\n",
    "    \n",
    "    plt.figure(figsize=(10,6))\n",
    "    plt.scatter(np.arange(len(movingAveA)),movingAveA)\n",
    "    plt.scatter(np.arange(len(movingAveB)),movingAveB)\n",
    "    \n",
    "mapScores()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Score:  2523.704\n",
      "Std Deviation:  1048.44\n",
      "Time elapsed:   438.53\n"
     ]
    }
   ],
   "source": [
    "def testAgent():\n",
    "    games = 500\n",
    "    \n",
    "    scores = np.zeros(games)\n",
    "    \n",
    "    for i in range(games):\n",
    "        scores[i] = (run(initializeEnv(),agent,memCount,True))\n",
    "    \n",
    "    print(\"Average Score: \", np.mean(scores))\n",
    "    print(\"Std Deviation: \", format(np.std(scores),'.2f'))\n",
    "    print(\"Time elapsed:  \", format(time.time() - startTime, '.2f'))\n",
    "    \n",
    "\n",
    "testAgent()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 256)               4352      \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 4)                 1028      \n",
      "=================================================================\n",
      "Total params: 136,964\n",
      "Trainable params: 136,964\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "agent.summary()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}