FlappyBird_Experiment_Wrapper-bruno-TPU-test.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FlappyBird Experiment Wrapper.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/LEChaney/ML-RayCast-Experiments/blob/master/FlappyBird_Experiment_Wrapper-bruno-TPU-test.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IzTlE0wWB2mn",
        "colab_type": "code",
        "outputId": "78251cef-b0e4-4a8d-c211-e2a0513b8511",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "# Dependencies\n",
        "%tensorflow_version"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Currently selected TF version: 1.x\n",
            "Available versions:\n",
            "* 1.x\n",
            "* 2.x\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gb2SzQ4_BJ0i",
        "colab_type": "code",
        "outputId": "2a373bde-22d2-4f55-803f-e26ac7b5f167",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "!pip install pygame\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pygame\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8e/24/ede6428359f913ed9cd1643dd5533aefeb5a2699cc95bea089de50ead586/pygame-1.9.6-cp36-cp36m-manylinux1_x86_64.whl (11.4MB)\n",
            "\u001b[K     |████████████████████████████████| 11.4MB 3.2MB/s \n",
            "\u001b[?25hInstalling collected packages: pygame\n",
            "Successfully installed pygame-1.9.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-uYauIb0H3KS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        " "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ilkVOEQoNeH9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "#import pygame\n",
        "os.environ[\"SDL_VIDEODRIVER\"] = \"dummy\"\n",
        "#pygame.init()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3t-fvH8M760",
        "colab_type": "code",
        "outputId": "4cd3023c-66ae-45c9-fb4f-fa1a3a034e87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "!git clone https://github.com/LEChaney/ML-RayCast-Experiments\n",
        "%cd ML-RayCast-Experiments/DeepLearningFlappyBird/\n",
        "# !python deep_q_network.py"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'ML-RayCast-Experiments'...\n",
            "remote: Enumerating objects: 164, done.\u001b[K\n",
            "remote: Counting objects: 100% (164/164), done.\u001b[K\n",
            "remote: Compressing objects: 100% (118/118), done.\u001b[K\n",
            "remote: Total 164 (delta 90), reused 103 (delta 44), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (164/164), 6.24 MiB | 24.87 MiB/s, done.\n",
            "Resolving deltas: 100% (90/90), done.\n",
            "/content/ML-RayCast-Experiments/DeepLearningFlappyBird\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5rapp_2Il7zh",
        "colab_type": "code",
        "outputId": "2fa9733d-127f-4455-8e6d-1a60131a3ed7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 834
        }
      },
      "source": [
        "#!/usr/bin/env python\n",
        "from __future__ import print_function\n",
        "\n",
        "import tensorflow as tf\n",
        "import cv2\n",
        "import sys\n",
        "sys.path.append(\"game/\")\n",
        "import wrapped_flappy_bird as game\n",
        "import random\n",
        "import numpy as np\n",
        "from collections import deque\n",
        "import os\n",
        "import contextlib\n",
        "\n",
        "@contextlib.contextmanager\n",
        "def dummy_context_mgr():\n",
        "    yield None\n",
        "\n",
        "GAME = 'bird' # the name of the game being played for log files\n",
        "ACTIONS = 2 # number of valid actions\n",
        "GAMMA = 0.99 # decay rate of past observations\n",
        "OBSERVE = 10000. # timesteps to observe before training\n",
        "EXPLORE = 1000000. # frames over which to anneal epsilon\n",
        "FINAL_EPSILON = 0.0001 # final value of epsilon\n",
        "INITIAL_EPSILON = 0.1 # starting value of epsilon\n",
        "REPLAY_MEMORY = 50000 # number of previous transitions to remember\n",
        "BATCH = 32 # size of minibatch\n",
        "FRAME_PER_ACTION = 1\n",
        "\n",
        "def weight_variable(shape):\n",
        "    initial = tf.truncated_normal(shape, stddev = 0.01)\n",
        "    return tf.Variable(initial)\n",
        "\n",
        "def bias_variable(shape):\n",
        "    initial = tf.constant(0.01, shape = shape)\n",
        "    return tf.Variable(initial)\n",
        "\n",
        "def conv2d(x, W, stride):\n",
        "    return tf.nn.conv2d(x, W, strides = [1, stride, stride, 1], padding = \"SAME\")\n",
        "\n",
        "def max_pool_2x2(x):\n",
        "    return tf.nn.max_pool(x, ksize = [1, 2, 2, 1], strides = [1, 2, 2, 1], padding = \"SAME\")\n",
        "\n",
        "def using_TPU():\n",
        "    if 'COLAB_TPU_ADDR' in os.environ:\n",
        "        return True\n",
        "    else:\n",
        "        return False\n",
        "\n",
        "if using_TPU():\n",
        "    TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']\n",
        "    resolver = tf.contrib.cluster_resolver.TPUClusterResolver(tpu=TPU_WORKER)\n",
        "    tf.contrib.distribute.initialize_tpu_system(resolver)\n",
        "    strategy = tf.contrib.distribute.TPUStrategy(resolver)\n",
        "    print('using TPU')\n",
        "      \n",
        "with strategy.scope() if using_TPU() else dummy_context_mgr():\n",
        "  \n",
        "    def createNetwork():\n",
        "        # network weights\n",
        "        W_conv1 = weight_variable([8, 8, 4, 32])\n",
        "        b_conv1 = bias_variable([32])\n",
        "\n",
        "        W_conv2 = weight_variable([4, 4, 32, 64])\n",
        "        b_conv2 = bias_variable([64])\n",
        "\n",
        "        W_conv3 = weight_variable([3, 3, 64, 64])\n",
        "        b_conv3 = bias_variable([64])\n",
        "\n",
        "        W_fc1 = weight_variable([1600, 512])\n",
        "        b_fc1 = bias_variable([512])\n",
        "\n",
        "        W_fc2 = weight_variable([512, ACTIONS])\n",
        "        b_fc2 = bias_variable([ACTIONS])\n",
        "\n",
        "        # input layer\n",
        "        s = tf.placeholder(\"float\", [None, 80, 80, 4])\n",
        "\n",
        "        # hidden layers\n",
        "        h_conv1 = tf.nn.relu(conv2d(s, W_conv1, 4) + b_conv1)\n",
        "        h_pool1 = max_pool_2x2(h_conv1)\n",
        "\n",
        "        h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2, 2) + b_conv2)\n",
        "        #h_pool2 = max_pool_2x2(h_conv2)\n",
        "\n",
        "        h_conv3 = tf.nn.relu(conv2d(h_conv2, W_conv3, 1) + b_conv3)\n",
        "        #h_pool3 = max_pool_2x2(h_conv3)\n",
        "\n",
        "        #h_pool3_flat = tf.reshape(h_pool3, [-1, 256])\n",
        "        h_conv3_flat = tf.reshape(h_conv3, [-1, 1600])\n",
        "\n",
        "        h_fc1 = tf.nn.relu(tf.matmul(h_conv3_flat, W_fc1) + b_fc1)\n",
        "\n",
        "        # readout layer\n",
        "        readout = tf.matmul(h_fc1, W_fc2) + b_fc2\n",
        "\n",
        "        return s, readout, h_fc1\n",
        "\n",
        "    @tf.function\n",
        "    def distributed_train_step(dataset_inputs):\n",
        "      per_replica_losses = strategy.experimental_run_v2(train_step,\n",
        "                                                      args=(dataset_inputs,))\n",
        "      return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,\n",
        "                           axis=None)\n",
        "  \n",
        "    def trainStep(a, y, readout):\n",
        "       a = tf.placeholder(\"float\", [None, ACTIONS])\n",
        "        y = tf.placeholder(\"float\", [None])\n",
        "        readout_action = tf.reduce_sum(tf.multiply(readout, a), reduction_indices=1)\n",
        "        cost = tf.reduce_mean(tf.square(y - readout_action))\n",
        "        optimizer = tf.train.AdamOptimizer(1e-3)\n",
        "        grads_and_vars = optimizer.compute_gradients(cost)\n",
        "        train_step = optimizer.apply_gradients(strategy, grads_and_vars)\n",
        "        \n",
        "    \n",
        "    def trainNetwork(s, readout, h_fc1):\n",
        "        # define the cost function\n",
        "       \n",
        "\n",
        "        # open up a game state to communicate with emulator\n",
        "        game_state = game.GameState()\n",
        "\n",
        "        # store the previous observations in replay memory\n",
        "        D = deque()\n",
        "\n",
        "        # printing\n",
        "        a_file = open(\"logs_\" + GAME + \"/readout.txt\", 'w')\n",
        "        h_file = open(\"logs_\" + GAME + \"/hidden.txt\", 'w')\n",
        "\n",
        "        # get the first state by doing nothing and preprocess the image to 80x80x4\n",
        "        do_nothing = np.zeros(ACTIONS)\n",
        "        do_nothing[0] = 1\n",
        "        x_t, r_0, terminal = game_state.frame_step(do_nothing)\n",
        "        x_t = cv2.cvtColor(cv2.resize(x_t, (80, 80)), cv2.COLOR_BGR2GRAY)\n",
        "        ret, x_t = cv2.threshold(x_t,1,255,cv2.THRESH_BINARY)\n",
        "        s_t = np.stack((x_t, x_t, x_t, x_t), axis=2)\n",
        "\n",
        "        # saving and loading networks\n",
        "        saver = tf.train.Saver()\n",
        "        sess.run(tf.initialize_all_variables())\n",
        "        checkpoint = tf.train.get_checkpoint_state(\"saved_networks\")\n",
        "        # if checkpoint and checkpoint.model_checkpoint_path:\n",
        "        #     saver.restore(sess, checkpoint.model_checkpoint_path)\n",
        "        #     print(\"Successfully loaded:\", checkpoint.model_checkpoint_path)\n",
        "        # else:\n",
        "        #     print(\"Could not find old network weights\")\n",
        "\n",
        "        # start training\n",
        "        epsilon = INITIAL_EPSILON\n",
        "        t = 0\n",
        "        while \"flappy bird\" != \"angry bird\":\n",
        "            # choose an action epsilon greedily\n",
        "            readout_t = readout.eval(feed_dict={s : [s_t]})[0]\n",
        "            a_t = np.zeros([ACTIONS])\n",
        "            action_index = 0\n",
        "            if t % FRAME_PER_ACTION == 0:\n",
        "                if random.random() <= epsilon:\n",
        "                    # print(\"----------Random Action----------\")\n",
        "                    action_index = random.randrange(ACTIONS)\n",
        "                    a_t[random.randrange(ACTIONS)] = 1\n",
        "                else:\n",
        "                    action_index = np.argmax(readout_t)\n",
        "                    a_t[action_index] = 1\n",
        "            else:\n",
        "                a_t[0] = 1 # do nothing\n",
        "\n",
        "            # scale down epsilon\n",
        "            if epsilon > FINAL_EPSILON and t > OBSERVE:\n",
        "                epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / EXPLORE\n",
        "\n",
        "            # run the selected action and observe next state and reward\n",
        "            x_t1_colored, r_t, terminal = game_state.frame_step(a_t)\n",
        "            x_t1 = cv2.cvtColor(cv2.resize(x_t1_colored, (80, 80)), cv2.COLOR_BGR2GRAY)\n",
        "            ret, x_t1 = cv2.threshold(x_t1, 1, 255, cv2.THRESH_BINARY)\n",
        "            x_t1 = np.reshape(x_t1, (80, 80, 1))\n",
        "            #s_t1 = np.append(x_t1, s_t[:,:,1:], axis = 2)\n",
        "            s_t1 = np.append(x_t1, s_t[:, :, :3], axis=2)\n",
        "\n",
        "            # store the transition in D\n",
        "            D.append((s_t, a_t, r_t, s_t1, terminal))\n",
        "            if len(D) > REPLAY_MEMORY:\n",
        "                D.popleft()\n",
        "\n",
        "            # only train if done observing\n",
        "            if t > OBSERVE:\n",
        "                # sample a minibatch to train on\n",
        "                minibatch = random.sample(D, BATCH)\n",
        "\n",
        "                # get the batch variables\n",
        "                s_j_batch = [d[0] for d in minibatch]\n",
        "                a_batch = [d[1] for d in minibatch]\n",
        "                r_batch = [d[2] for d in minibatch]\n",
        "                s_j1_batch = [d[3] for d in minibatch]\n",
        "\n",
        "                y_batch = []\n",
        "                readout_j1_batch = readout.eval(feed_dict = {s : s_j1_batch})\n",
        "                for i in range(0, len(minibatch)):\n",
        "                    terminal = minibatch[i][4]\n",
        "                    # if terminal, only equals reward\n",
        "                    if terminal:\n",
        "                        y_batch.append(r_batch[i])\n",
        "                    else:\n",
        "                        y_batch.append(r_batch[i] + GAMMA * np.max(readout_j1_batch[i]))\n",
        "\n",
        "                # perform gradient step\n",
        "                train_step.run(feed_dict = {\n",
        "                    y : y_batch,\n",
        "                    a : a_batch,\n",
        "                    s : s_j_batch}\n",
        "                )\n",
        "\n",
        "            # update the old values\n",
        "            s_t = s_t1\n",
        "            t += 1\n",
        "\n",
        "            # save progress every 10000 iterations\n",
        "            if t % 10000 == 0:\n",
        "                saver.save(sess, 'saved_networks/' + GAME + '-dqn', global_step = t)\n",
        "\n",
        "            # print info\n",
        "            state = \"\"\n",
        "            if t <= OBSERVE:\n",
        "                state = \"observe\"\n",
        "            elif t > OBSERVE and t <= OBSERVE + EXPLORE:\n",
        "                state = \"explore\"\n",
        "            else:\n",
        "                state = \"train\"\n",
        "\n",
        "            if t % 10000 == 0:\n",
        "                print(\"TIMESTEP\", t, \"/ STATE\", state, \\\n",
        "                    \"/ EPSILON\", epsilon, \"/ ACTION\", action_index, \"/ REWARD\", r_t, \\\n",
        "                    \"/ Q_MAX %e\" % np.max(readout_t))\n",
        "            # write info to files\n",
        "            '''\n",
        "            if t % 10000 <= 100:\n",
        "                a_file.write(\",\".join([str(x) for x in readout_t]) + '\\n')\n",
        "                h_file.write(\",\".join([str(x) for x in h_fc1.eval(feed_dict={s:[s_t]})[0]]) + '\\n')\n",
        "                cv2.imwrite(\"logs_tetris/frame\" + str(t) + \".png\", x_t1)\n",
        "        \n",
        "            '''\n",
        "\n",
        "    def playGame():\n",
        "        sess = tf.InteractiveSession()\n",
        "        s, readout, h_fc1 = createNetwork()\n",
        "        trainNetwork(s, readout, h_fc1)\n",
        "\n",
        "    def main():\n",
        "        playGame()\n",
        "\n",
        "# if __name__ == \"__main__\":\n",
        "    main()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "pygame 1.9.6\n",
            "Hello from the pygame community. https://www.pygame.org/contribute.html\n",
            "WARNING:tensorflow:\n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "INFO:tensorflow:Initializing the TPU system.\n",
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "INFO:tensorflow:Querying Tensorflow master (grpc://10.4.62.194:8470) for TPU system metadata.\n",
            "INFO:tensorflow:Found TPU system:\n",
            "INFO:tensorflow:*** Num TPU Cores: 8\n",
            "INFO:tensorflow:*** Num TPU Workers: 1\n",
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 16839804752634262363)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 13389748452131715668)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 18426361293801205529)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 10379285470724107230)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4910844846988266689)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 12164834416154457879)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 13301382548551886825)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 7150500014637568344)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 944840208366264296)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 8589934592, 2476565381499566037)\n",
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 9294028492147601464)\n",
            "using TPU\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-9c9ca55a236a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[0;31m# if __name__ == \"__main__\":\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 236\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-5-9c9ca55a236a>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 233\u001b[0;31m         \u001b[0mplayGame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[0;31m# if __name__ == \"__main__\":\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-5-9c9ca55a236a>\u001b[0m in \u001b[0;36mplayGame\u001b[0;34m()\u001b[0m\n\u001b[1;32m    228\u001b[0m         \u001b[0msess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInteractiveSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m         \u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreadout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh_fc1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreateNetwork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 230\u001b[0;31m         \u001b[0mtrainNetwork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreadout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh_fc1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-5-9c9ca55a236a>\u001b[0m in \u001b[0;36mtrainNetwork\u001b[0;34m(s, readout, h_fc1)\u001b[0m\n\u001b[1;32m    103\u001b[0m         \u001b[0moptimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAdamOptimizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1e-3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m         \u001b[0mgrads_and_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcost\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 105\u001b[0;31m         \u001b[0mtrain_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_distributed_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstrategy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrads_and_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m         \u001b[0;31m# open up a game state to communicate with emulator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/optimizer.py\u001b[0m in \u001b[0;36m_distributed_apply\u001b[0;34m(self, distribution, grads_and_vars, global_step, name)\u001b[0m\n\u001b[1;32m    663\u001b[0m     \"\"\"\n\u001b[1;32m    664\u001b[0m     reduced_grads = distribution.extended.batch_reduce_to(\n\u001b[0;32m--> 665\u001b[0;31m         ds_reduce_util.ReduceOp.SUM, grads_and_vars)\n\u001b[0m\u001b[1;32m    666\u001b[0m     \u001b[0mvar_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mv\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgrads_and_vars\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    667\u001b[0m     \u001b[0mgrads_and_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduced_grads\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36mbatch_reduce_to\u001b[0;34m(self, reduce_op, value_destination_pairs)\u001b[0m\n\u001b[1;32m   1252\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstring_types\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1253\u001b[0m       \u001b[0mreduce_op\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreduce_util\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mReduceOp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduce_op\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1254\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_reduce_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue_destination_pairs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1255\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1256\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_batch_reduce_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue_destination_pairs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36m_batch_reduce_to\u001b[0;34m(self, reduce_op, value_destination_pairs)\u001b[0m\n\u001b[1;32m   1257\u001b[0m     return [\n\u001b[1;32m   1258\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduce_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestinations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1259\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvalue_destination_pairs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1260\u001b[0m     ]\n\u001b[1;32m   1261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1257\u001b[0m     return [\n\u001b[1;32m   1258\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduce_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestinations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1259\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvalue_destination_pairs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1260\u001b[0m     ]\n\u001b[1;32m   1261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36mreduce_to\u001b[0;34m(self, reduce_op, value, destinations)\u001b[0m\n\u001b[1;32m   1231\u001b[0m     assert (reduce_op == reduce_util.ReduceOp.SUM or\n\u001b[1;32m   1232\u001b[0m             reduce_op == reduce_util.ReduceOp.MEAN)\n\u001b[0;32m-> 1233\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reduce_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestinations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1235\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_reduce_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestinations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/tpu_strategy.py\u001b[0m in \u001b[0;36m_reduce_to\u001b[0;34m(self, reduce_op, value, destinations)\u001b[0m\n\u001b[1;32m    520\u001b[0m       \u001b[0;31m# be 0.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    521\u001b[0m       return cross_device_ops_lib.reduce_non_distributed_value(\n\u001b[0;32m--> 522\u001b[0;31m           reduce_op, self._device_map, value, destinations)\n\u001b[0m\u001b[1;32m    523\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    524\u001b[0m     \u001b[0mdevices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcross_device_ops_lib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_devices_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdestinations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/cross_device_ops.py\u001b[0m in \u001b[0;36mreduce_non_distributed_value\u001b[0;34m(reduce_op, device_map, value, destinations)\u001b[0m\n\u001b[1;32m     92\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mdevice_map\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_replicas_in_graph\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m     raise ValueError(\"A non-DistributedValues value %s cannot be reduced with \"\n\u001b[0;32m---> 94\u001b[0;31m                      \"the given reduce op %s.\" % (value, reduce_op))\n\u001b[0m\u001b[1;32m     95\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0msimple_broadcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestinations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: A non-DistributedValues value Tensor(\"gradients/Conv2D_grad/tuple/control_dependency_1:0\", shape=(8, 8, 4, 32), dtype=float32) cannot be reduced with the given reduce op ReduceOp.SUM."
          ]
        }
      ]
    }
  ]
}