RNNQLEARNER.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import theano as theano\n",
    "import theano.tensor as T\n",
    "from theano.gradient import grad_clip\n",
    "import numpy as np\n",
    "import time\n",
    "import operator\n",
    "import sys\n",
    "from time import time\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class GRUTheano:\n",
    "    \n",
    "    def __init__(self, x_dim,y_dim, hidden_dim=128, bptt_truncate=-1):\n",
    "        # Assign instance variables\n",
    "        self.x_dim = x_dim\n",
    "        self.y_dim = y_dim\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.bptt_truncate = bptt_truncate\n",
    "        # Initialize the network parameters\n",
    "        E = np.random.uniform(-np.sqrt(1./x_dim), np.sqrt(1./x_dim), (hidden_dim, x_dim))\n",
    "        U = np.random.uniform(-np.sqrt(1./hidden_dim), np.sqrt(1./hidden_dim), (6, hidden_dim, hidden_dim))\n",
    "        W = np.random.uniform(-np.sqrt(1./hidden_dim), np.sqrt(1./hidden_dim), (6, hidden_dim, hidden_dim))\n",
    "        V = np.random.uniform(-np.sqrt(1./hidden_dim), np.sqrt(1./hidden_dim), (y_dim, hidden_dim))\n",
    "        b = np.zeros((6, hidden_dim))\n",
    "        c = np.zeros(y_dim)\n",
    "        # Theano: Created shared variables\n",
    "        self.E = theano.shared(name='E', value=E.astype(theano.config.floatX))\n",
    "        self.U = theano.shared(name='U', value=U.astype(theano.config.floatX))\n",
    "        self.W = theano.shared(name='W', value=W.astype(theano.config.floatX))\n",
    "        self.V = theano.shared(name='V', value=V.astype(theano.config.floatX))\n",
    "        self.b = theano.shared(name='b', value=b.astype(theano.config.floatX))\n",
    "        self.c = theano.shared(name='c', value=c.astype(theano.config.floatX))\n",
    "        # SGD / rmsprop: Initialize parameters\n",
    "        self.mE = theano.shared(name='mE', value=np.zeros(E.shape).astype(theano.config.floatX))\n",
    "        self.mU = theano.shared(name='mU', value=np.zeros(U.shape).astype(theano.config.floatX))\n",
    "        self.mV = theano.shared(name='mV', value=np.zeros(V.shape).astype(theano.config.floatX))\n",
    "        self.mW = theano.shared(name='mW', value=np.zeros(W.shape).astype(theano.config.floatX))\n",
    "        self.mb = theano.shared(name='mb', value=np.zeros(b.shape).astype(theano.config.floatX))\n",
    "        self.mc = theano.shared(name='mc', value=np.zeros(c.shape).astype(theano.config.floatX))\n",
    "        # We store the Theano graph here\n",
    "        self.theano = {}\n",
    "        self.__theano_build__()\n",
    "    \n",
    "    def __theano_build__(self):\n",
    "        E, V, U, W, b, c = self.E, self.V, self.U, self.W, self.b, self.c\n",
    "        \n",
    "        x = T.dmatrix('x')\n",
    "        y = T.dmatrix('y')\n",
    "        \n",
    "        def forward_prop_step(x_t, s_t1_prev, s_t2_prev):\n",
    "            # This is how we calculated the hidden state in a simple RNN. No longer!\n",
    "            # s_t = T.tanh(U[:,x_t] + W.dot(s_t1_prev))\n",
    "            \n",
    "            # Word embedding layer\n",
    "            x_e = E.dot(x_t)\n",
    "            \n",
    "            # GRU Layer 1\n",
    "            z_t1 = T.nnet.sigmoid(U[0].dot(x_e) + W[0].dot(s_t1_prev) + b[0])\n",
    "            r_t1 = T.nnet.sigmoid(U[1].dot(x_e) + W[1].dot(s_t1_prev) + b[1])\n",
    "            c_t1 = T.tanh(U[2].dot(x_e) + W[2].dot(s_t1_prev * r_t1) + b[2])\n",
    "            s_t1 = (T.ones_like(z_t1) - z_t1) * c_t1 + z_t1 * s_t1_prev\n",
    "            \n",
    "            # GRU Layer 2\n",
    "            z_t2 = T.nnet.sigmoid(U[3].dot(s_t1) + W[3].dot(s_t2_prev) + b[3])\n",
    "            r_t2 = T.nnet.sigmoid(U[4].dot(s_t1) + W[4].dot(s_t2_prev) + b[4])\n",
    "            c_t2 = T.tanh(U[5].dot(s_t1) + W[5].dot(s_t2_prev * r_t2) + b[5])\n",
    "            s_t2 = (T.ones_like(z_t2) - z_t2) * c_t2 + z_t2 * s_t2_prev\n",
    "            \n",
    "            # Final output calculation\n",
    "            o_t = T.nnet.relu(V.dot(s_t2) + c)\n",
    "\n",
    "            return [o_t, s_t1, s_t2]\n",
    "        \n",
    "        [o, s, s2], updates = theano.scan(\n",
    "            forward_prop_step,\n",
    "            sequences=x,\n",
    "            truncate_gradient=self.bptt_truncate,\n",
    "            outputs_info=[None, \n",
    "                          dict(initial=T.zeros(self.hidden_dim)),\n",
    "                          dict(initial=T.zeros(self.hidden_dim))])\n",
    "        \n",
    "        prediction = T.argmax(o, axis=1)\n",
    "        o_error = T.sum(T.pow(o-y,2))\n",
    "        \n",
    "        # Total cost (could add regularization here)\n",
    "        cost = o_error\n",
    "        \n",
    "        # Gradients\n",
    "        dE = T.grad(cost, E)\n",
    "        dU = T.grad(cost, U)\n",
    "        dW = T.grad(cost, W)\n",
    "        db = T.grad(cost, b)\n",
    "        dV = T.grad(cost, V)\n",
    "        dc = T.grad(cost, c)\n",
    "        \n",
    "        # Assign functions\n",
    "        self.predict = theano.function([x], o)\n",
    "        self.predict_class = theano.function([x], prediction)\n",
    "        self.ce_error = theano.function([x, y], cost)\n",
    "        self.bptt = theano.function([x, y], [dE, dU, dW, db, dV, dc])\n",
    "        \n",
    "        # SGD parameters\n",
    "        learning_rate = T.scalar('learning_rate')\n",
    "        decay = T.scalar('decay')\n",
    "        \n",
    "        # rmsprop cache updates\n",
    "        mE = decay * self.mE + (1 - decay) * dE ** 2\n",
    "        mU = decay * self.mU + (1 - decay) * dU ** 2\n",
    "        mW = decay * self.mW + (1 - decay) * dW ** 2\n",
    "        mV = decay * self.mV + (1 - decay) * dV ** 2\n",
    "        mb = decay * self.mb + (1 - decay) * db ** 2\n",
    "        mc = decay * self.mc + (1 - decay) * dc ** 2\n",
    "        \n",
    "        self.sgd_step = theano.function(\n",
    "            [x, y, learning_rate, theano.In(decay, value=0.9)],\n",
    "            [], \n",
    "            updates=[(E, E - learning_rate * dE / T.sqrt(mE + 1e-6)),\n",
    "                     (U, U - learning_rate * dU / T.sqrt(mU + 1e-6)),\n",
    "                     (W, W - learning_rate * dW / T.sqrt(mW + 1e-6)),\n",
    "                     (V, V - learning_rate * dV / T.sqrt(mV + 1e-6)),\n",
    "                     (b, b - learning_rate * db / T.sqrt(mb + 1e-6)),\n",
    "                     (c, c - learning_rate * dc / T.sqrt(mc + 1e-6)),\n",
    "                     (self.mE, mE),\n",
    "                     (self.mU, mU),\n",
    "                     (self.mW, mW),\n",
    "                     (self.mV, mV),\n",
    "                     (self.mb, mb),\n",
    "                     (self.mc, mc)\n",
    "                    ])\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def num_to_move(num):\n",
    "    if num == 0:\n",
    "        return \"bet 0-25\"\n",
    "    elif num == 1:\n",
    "        return \"bet 25-50\"\n",
    "    elif num == 2:\n",
    "        return \"bet 50-75\"\n",
    "    elif num == 3:\n",
    "        return \"bet 75-100\"\n",
    "    elif num == 4:\n",
    "        return \"bet 100-125\"\n",
    "    elif num == 5:\n",
    "        return \"bet 125-150\"\n",
    "    elif num == 6:\n",
    "        return \"bet 150-175\"\n",
    "    elif num == 7:\n",
    "        return \"bet 175-200\"\n",
    "def save_model_parameters_theano(model, outfile):\n",
    "    np.savez(outfile,\n",
    "        E=model.E.get_value(),\n",
    "        U=model.U.get_value(),\n",
    "        W=model.W.get_value(),\n",
    "        V=model.V.get_value(),\n",
    "        b=model.b.get_value(),\n",
    "        c=model.c.get_value())\n",
    "    print \"Saved model parameters to %s.\" % outfile\n",
    "def load_model_parameters_theano(path, modelClass=GRUTheano):\n",
    "    npzfile = np.load(path)\n",
    "    E, U, W, V, b, c = npzfile[\"E\"], npzfile[\"U\"], npzfile[\"W\"], npzfile[\"V\"], npzfile[\"b\"], npzfile[\"c\"]\n",
    "    hidden_dim, x_dim = E.shape[0], E.shape[1]\n",
    "    y_dim,hidden_dim = V.shape[0], V.shape[1]\n",
    "    print \"Building model model from %s with hidden_dim=%d x_dim=%d y_dim=%d \" % (path, hidden_dim, x_dim,y_dim)\n",
    "    sys.stdout.flush()\n",
    "    model = modelClass(x_dim, y_dim,hidden_dim=hidden_dim)\n",
    "    model.E.set_value(E)\n",
    "    model.U.set_value(U)\n",
    "    model.W.set_value(W)\n",
    "    model.V.set_value(V)\n",
    "    model.b.set_value(b)\n",
    "    model.c.set_value(c)\n",
    "    return model\n",
    "def predict_hand(hand):\n",
    "    states = np.array([])\n",
    "    moves = []\n",
    "    for state,action,reward,done in hand:\n",
    "        state = np.append(state,action)\n",
    "        if len(states) == 0:\n",
    "            states = np.array([state])\n",
    "        else:\n",
    "            states = np.append(states,np.array([state]),axis=0) \n",
    "        moves += [model.predict(states)[-1]]\n",
    "    return moves\n",
    "def predict_hand_moves(hand):\n",
    "    states = np.array([])\n",
    "    moves = []\n",
    "    for state,action,reward,done in hand:\n",
    "        state = np.append(state,action)\n",
    "        if len(states) == 0:\n",
    "            states = np.array([state])\n",
    "        else:\n",
    "            states = np.append(states,np.array([state]),axis=0) \n",
    "        moves += [num_to_move(np.argmax(model.predict(states)[-1]))]\n",
    "## HELPER FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from time import time\n",
    "from time import sleep\n",
    "import commands\n",
    "import os\n",
    "count = 0\n",
    "\n",
    "experience_replay = [] ## MEMORIES OF EXPERIENCES WITH HIGH REWARD MAGNITUDES\n",
    "recent_memory = []     ## RECENT MEMORIES FROM PARSED FILES\n",
    "                       ## SAMPLE RANSOMLY FROM THIS TO REDUCE OVERESTIMATION\n",
    "recent_memory_size = 400\n",
    "replay_memory = 100\n",
    "states_seen = 0\n",
    "\n",
    "memory = 30\n",
    "counts = {}\n",
    "lr = 1e-5\n",
    "gamma =.95\n",
    "reward_length = 30 # GIVE REWARD AS AVERAGE OVER MANY STATES TO REDUCE RISKY BEHAVIOR\n",
    "\n",
    "def train():\n",
    "    \n",
    "    ## ITERATE CONFIG FILES TO TRAIN ON MULTIPLE OTHER BOTS WITH DIFFERENT STRATEGIES\n",
    "    for config in glob.glob(\"/Users/michaelgump/pokerbots-2017/config_files/*.txt\"):\n",
    "        \n",
    "        ## WRITE IT TO THE MAIN DIRECTORY SO THE ENGINE WIL RUN IT\n",
    "        with open(config,'rb') as conf:\n",
    "            content = conf.readlines()\n",
    "            text_file = open(\"/Users/michaelgump/pokerbots-2017/config.txt\", \"wb\")\n",
    "            for line in content:\n",
    "                text_file.write(line)\n",
    "            text_file.close()\n",
    "        \n",
    "        s=commands.getstatusoutput('cd /Users/michaelgump/pokerbots-2017/;java -jar engine.jar -Q')\n",
    "        sleep(20)\n",
    "        files = glob.glob(\"/Users/michaelgump/pokerbots-2017/selftraining/*.p\")\n",
    "        \n",
    "        last_rewards = []\n",
    "        \n",
    "        # ITERATE PARSED GAMES\n",
    "        for f in files:\n",
    "            print f\n",
    "            s= time()\n",
    "            states,rewards = pickle.load(open(f,\"rb\"))\n",
    "\n",
    "            train_states = np.array([])\n",
    "            train_targets = np.array([])\n",
    "\n",
    "            for i in range(len(states)):\n",
    "\n",
    "                state= states[i]\n",
    "                r,action,done = rewards[i]\n",
    "\n",
    "                if len(train_states) == 0:\n",
    "                    train_states = np.array([state])\n",
    "                    target = model.predict(states)[-1]\n",
    "                    train_targets = np.array([target])                \n",
    "                else:\n",
    "                    qval = model.predict(states)[-1]\n",
    "\n",
    "                    if len(train_states)<memory:\n",
    "                        train_states = np.append(train_states,np.array([state]),axis=0)\n",
    "                    else:\n",
    "                        train_states = np.append(train_states[1:],np.array([state]),axis=0)\n",
    "\n",
    "                    newQ = model_value.predict(states)[-1]  # USE A VERSION OF THE NETWORK MANY ITERATIONS AGO\n",
    "                                                            # FOR ESTIMATING VALUE OF NEXT STATE TO REDUCE\n",
    "                                                            # ESTIMATION\n",
    "                            \n",
    "                    maxQ = np.max(newQ)\n",
    "\n",
    "                    update = 0\n",
    "                    \n",
    "                    last_rewards = last_rewards[-reward_length:]+[float(reward)]\n",
    "                    reward = float(sum(last_rewards))/reward_length\n",
    "                    \n",
    "                    if done:\n",
    "                        update = reward\n",
    "                    else:\n",
    "                        update = reward+gamma*maxQ\n",
    "\n",
    "                    text = num_to_move(action)\n",
    "                    \n",
    "                    if text in counts:\n",
    "                        counts[text]+=1\n",
    "                    else:\n",
    "                        counts[text]=1\n",
    "\n",
    "                    target = np.zeros(8)\n",
    "                    target[:] = qval[:]\n",
    "                    target[action] = float(update)\n",
    "                    \n",
    "                    if len(train_targets)<memory:\n",
    "                        train_targets = np.append(train_targets,np.array([target]),axis=0)\n",
    "                    else:\n",
    "                        train_targets = np.append(train_targets[1:],np.array([target]),axis=0)\n",
    "\n",
    "                    if len(experience_replay)>replay_memory:\n",
    "                        experience_replay.sort(key=lambda x:x[2])\n",
    "                        if float(r)>experience_replay[0][2]:\n",
    "                            experience_replay = experience_replay[1:]+[(train_states,train_targets,reward)]\n",
    "                    else:\n",
    "                        experience_replay += [(train_states,train_targets,reward)]\n",
    "                    states_seen += 1\n",
    "                recent_memory += [(train_states,train_targets)]\n",
    "            \n",
    "            if len(recent_memory)>recent_memory_size: ## SAMPLE RECENT MEMORY RANDOMLY\n",
    "                for i in range(len(recent_memory)-recent_memory_size):\n",
    "                    inputs,outputs, = recent_memory.pop(np.random.randint(len(recent_memory)))\n",
    "                    model.sgd_step(inputs,outputs,lr)\n",
    "                    \n",
    "            if states_seen > 500: ## DONT START REPLAYING EXPERIENCES UNTIL A MEMORY HAS BEEN ASSEMBLED\n",
    "                exp_for_replay = experience_replay[np.random.randint(len(experience_replay))]\n",
    "                inputs,outputs,r = exp_for_replay\n",
    "                model.sgd_step(inputs,outputs,lr)\n",
    "                \n",
    "            os.remove(f)\n",
    "            \n",
    "    save_model_parameters_theano(model,'/Users/michaelgump/pokerbots-2017/modelbottest/model.npz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}