TradingActorCritic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iOrzvk2LHhO9"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import math, random\n",
    "from scipy import signal\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.autograd as autograd\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "q2nqynZOHhPL",
    "outputId": "9539a2ea-3763-425e-ee2f-bfad76dcf310"
   },
   "outputs": [],
   "source": [
    "torch.manual_seed(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HuuW7gWPHhPY"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('aapl.us.txt').iloc[6000:8001]\n",
    "apl_open = np.array(df['Open'])\n",
    "apl_close = np.array(df['Close'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "FfySDzCzHhPg",
    "outputId": "c9b8e714-a4bd-4e1c-8695-deddfce721c5"
   },
   "outputs": [],
   "source": [
    "apl_open = signal.detrend(apl_open)\n",
    "apl_close = signal.detrend(apl_close)\n",
    "print(apl_open.min(), apl_close.min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8peEalZHHhPo"
   },
   "outputs": [],
   "source": [
    "apl_open += (-apl_open.min() + 1)\n",
    "apl_close += (-apl_close.min() + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "wMBDqtpyHhPv",
    "outputId": "855879b9-fefd-4a57-c359-fe86aeeaa679"
   },
   "outputs": [],
   "source": [
    "print(apl_open.min(), apl_close.min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ij0sLUKNHhP3"
   },
   "outputs": [],
   "source": [
    "apl_open_orig = np.array(df['Open'])\n",
    "apl_close_orig = np.array(df['Close'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HDpxF3HUHhP7"
   },
   "outputs": [],
   "source": [
    "class Environment:\n",
    "    def __init__(self, starting_cash=200, randomize_cash=0, starting_shares=0,\n",
    "                 randomize_shares=0, max_stride=5, series_length=200,\n",
    "                 starting_point=0, inaction_penalty=0):\n",
    "        self.starting_cash = starting_cash\n",
    "        self.randomize_cash = randomize_cash\n",
    "        self.starting_shares = starting_shares\n",
    "        self.randomize_shares = randomize_shares\n",
    "        \n",
    "        self.starting_cash = max(int(np.random.normal(self.starting_cash, self.randomize_cash)), 0)\n",
    "        self.series_length = series_length\n",
    "        self.starting_point = starting_point\n",
    "        self.cur_timestep = self.starting_point\n",
    "        \n",
    "        self.state = torch.FloatTensor(torch.zeros(5)).cuda()\n",
    "        self.state[0] = max(int(np.random.normal(self.starting_shares, self.randomize_shares)), 0)\n",
    "        self.state[1] = self.starting_cash\n",
    "        self.state[2] = apl_open[self.cur_timestep]\n",
    "        self.starting_portfolio_value = self.portfolio_value()\n",
    "        self.state[3] = self.starting_portfolio_value\n",
    "        self.state[4] = self.five_day_window()\n",
    "        \n",
    "        self.max_stride = max_stride\n",
    "        self.stride = self.max_stride\n",
    "        \n",
    "        self.done = False\n",
    "        self.inaction_penalty = inaction_penalty\n",
    "        \n",
    "    def portfolio_value(self):\n",
    "        return ((self.state[0] * apl_close[self.cur_timestep]) + self.state[1])\n",
    "    \n",
    "    def next_opening_price(self):\n",
    "        next_timestep = self.cur_timestep + self.stride\n",
    "        return apl_open[next_timestep]\n",
    "    \n",
    "    def five_day_window(self):\n",
    "        step = self.cur_timestep\n",
    "        if step < 5:\n",
    "            return apl_open[0]\n",
    "        apl5 = apl_open[step-5:step].mean()\n",
    "        return apl5\n",
    "    \n",
    "    def step(self, action):\n",
    "        action = [action, 1]\n",
    "        cur_timestep = self.cur_timestep\n",
    "        ts_left = self.series_length - (cur_timestep - self.starting_point)\n",
    "        retval = None\n",
    "        cur_value = self.portfolio_value()\n",
    "        gain = cur_value - self.starting_portfolio_value\n",
    "        \n",
    "        if cur_timestep >= self.starting_point + self.series_length * self.stride:\n",
    "            next_opening = self.next_opening_price()\n",
    "            next_five_day = self.five_day_window()\n",
    "            new_state = [self.state[0], self.state[1], next_opening, cur_value, next_five_day]\n",
    "            self.state = new_state\n",
    "            return new_state, cur_value + gain, True, {'msg': 'done'}\n",
    "        \n",
    "        if action[0] == 2:\n",
    "            next_opening = self.next_opening_price()\n",
    "            next_five_day = self.five_day_window()\n",
    "            new_state = [self.state[0], self.state[1], next_opening, cur_value, next_five_day]\n",
    "            self.state = new_state\n",
    "            retval = new_state, gain - self.inaction_penalty - ts_left, False, {'msg': 'nothing'}\n",
    "            \n",
    "        if action[0] == 0:\n",
    "            if action[1] * apl_open[cur_timestep] > self.state[1]:\n",
    "                next_opening = self.next_opening_price()\n",
    "                next_five_day = self.five_day_window()\n",
    "                new_state = [self.state[0], self.state[1], next_opening, cur_value, next_five_day]\n",
    "                self.state = new_state\n",
    "                retval = new_state, -ts_left + gain / 2, True, {'msg': 'bankrupt'}\n",
    "            else:\n",
    "                apl_shares = self.state[0] + action[1]\n",
    "                cash_spent = action[1] * apl_open[cur_timestep] * 1.1\n",
    "                next_opening = self.next_opening_price()\n",
    "                next_five_day = self.five_day_window()\n",
    "                new_state = [apl_shares, self.state[1] - cash_spent, next_opening, cur_value, next_five_day]\n",
    "                self.state = new_state\n",
    "                retval = new_state, gain + self.inaction_penalty - ts_left, False, {'msg': 'bought stocks'}\n",
    "                \n",
    "        if action[0] == 1:\n",
    "            if action[1] > self.state[0]:\n",
    "                next_opening = self.next_opening_price()\n",
    "                next_five_day = self.five_day_window()\n",
    "                new_state = [self.state[0], self.state[1], next_opening, cur_value, next_five_day]\n",
    "                self.state = new_state\n",
    "                retval = new_state, -ts_left + gain / 2, True, {'msg': 'sold more than available'}\n",
    "            else:\n",
    "                apl_shares = self.state[0] - action[1]\n",
    "                cash_gained = action[1] * apl_open[cur_timestep] * 0.9\n",
    "                next_opening = self.next_opening_price()\n",
    "                next_five_day = self.five_day_window()\n",
    "                new_state = [apl_shares, self.state[1] + cash_gained, next_opening, cur_value, next_five_day]\n",
    "                self.state = new_state\n",
    "                retval = new_state, gain + self.inaction_penalty - ts_left, False, {'msg': 'sold stocks'}\n",
    "                \n",
    "        self.cur_timestep += self.stride\n",
    "        return retval\n",
    "    \n",
    "    def reset(self):\n",
    "        self.starting_cash = max(int(np.random.normal(self.starting_cash, self.randomize_cash)), 0.)\n",
    "        self.cur_timestep = self.starting_point\n",
    "        \n",
    "        self.state[0] = max(int(np.random.normal(self.starting_shares, self.randomize_shares)), 0)\n",
    "        self.state[1] = self.starting_cash\n",
    "        self.state[2] = apl_open[self.cur_timestep]\n",
    "        self.state[3] = self.starting_portfolio_value\n",
    "        self.state[4] = self.five_day_window()\n",
    "        \n",
    "        self.done = False\n",
    "        return self.state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l1NiHHDPHhQC"
   },
   "outputs": [],
   "source": [
    "class Policy(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Policy, self).__init__()\n",
    "        self.input_layer = nn.Linear(5, 128)\n",
    "        self.hidden_1 = nn.Linear(128, 128)\n",
    "        self.hidden_2 = nn.Linear(32, 31)\n",
    "        self.hidden_state = torch.tensor(torch.zeros(2, 1, 32)).cuda()\n",
    "        self.rnn = nn.GRU(128, 32, 2)\n",
    "        self.action_head = nn.Linear(31, 3)\n",
    "        self.value_head = nn.Linear(31, 1)\n",
    "        self.saved_actions = []\n",
    "        self.rewards = []\n",
    "        \n",
    "    def reset_hidden(self):\n",
    "        self.hidden_state = torch.tensor(torch.zeros(2, 1, 32)).cuda()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = torch.tensor(x).cuda()\n",
    "        x = torch.sigmoid(self.input_layer(x))\n",
    "        x = torch.tanh(self.hidden_1(x))\n",
    "        x, self.hidden_state = self.rnn(x.view(1, -1, 128), self.hidden_state.data)\n",
    "        x = F.relu(self.hidden_2(x.squeeze()))\n",
    "        action_scores = self.action_head(x)\n",
    "        state_values = self.value_head(x)\n",
    "        return F.softmax(action_scores, dim=-1), state_values\n",
    "    \n",
    "    def act(self, state):\n",
    "        probs, state_value = self.forward(state)\n",
    "        m = Categorical(probs)\n",
    "        action = m.sample()\n",
    "        if action == 1 and state[0] < 1:\n",
    "            action = torch.LongTensor([2]).squeeze().cuda()\n",
    "        self.saved_actions.append((m.log_prob(action), state_value))\n",
    "        return action.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74
    },
    "colab_type": "code",
    "id": "c3u0hi2EHhQI",
    "outputId": "20699faa-875c-4aca-d568-501215b4d76b"
   },
   "outputs": [],
   "source": [
    "env = Environment(starting_cash=1000, randomize_cash=100, starting_shares=100, \n",
    "                  randomize_shares=10, max_stride=4, series_length=499)\n",
    "model = Policy().cuda()\n",
    "optimizer = optim.Adam(model.parameters(), lr=4e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9_OrRoHKHhQP"
   },
   "outputs": [],
   "source": [
    "env.reset()\n",
    "del model.rewards[:]\n",
    "del model.saved_actions[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FZsrOFP1HhQX"
   },
   "outputs": [],
   "source": [
    "gamma = 0.8\n",
    "log_interval = 40\n",
    "running_reward = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZmO_39lpHhQc"
   },
   "outputs": [],
   "source": [
    "def finish_episode():\n",
    "    R = 0\n",
    "    saved_actions = model.saved_actions\n",
    "    policy_losses = []\n",
    "    value_losses = []\n",
    "    rewards = []\n",
    "    for r in model.rewards[::-1]:\n",
    "        R = r + (gamma * R)\n",
    "        rewards.insert(0, R)\n",
    "    rewards = torch.tensor(rewards)\n",
    "    epsilon = (torch.rand(1) / 1e4) - 5e-5\n",
    "    rewards += epsilon\n",
    "    \n",
    "    for (log_prob, value), r in zip(saved_actions, rewards):\n",
    "        reward = torch.tensor(r - value.item()).cuda()\n",
    "        policy_losses.append(-log_prob * reward)\n",
    "        value_losses.append(F.smooth_l1_loss(value, torch.tensor([r]).cuda()))\n",
    "        \n",
    "    optimizer.zero_grad()\n",
    "    loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum()\n",
    "    loss = torch.clamp(loss, -1e-5, 1e5)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    del model.rewards[:]\n",
    "    del model.saved_actions[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 822
    },
    "colab_type": "code",
    "id": "E_gzYyeSHhQj",
    "outputId": "9d67e610-dcea-47a9-996e-5162db62c569"
   },
   "outputs": [],
   "source": [
    "for episode in range(0, 4000):\n",
    "    state = env.reset()\n",
    "    reward = 0\n",
    "    done = False\n",
    "    msg = None\n",
    "    while not done:\n",
    "        action = model.act(state)\n",
    "        state, reward, done, msg = env.step(action)\n",
    "        model.rewards.append(reward)\n",
    "        if done:\n",
    "            break\n",
    "    running_reward = running_reward * (1 - 1 / log_interval) + reward * 1 / log_interval\n",
    "    finish_episode()\n",
    "    \n",
    "    if msg['msg'] == 'done' and running_reward > 300 and env.portfolio_value() > env.starting_portfolio_value:\n",
    "        print('Early stop: ', int(reward))\n",
    "        break\n",
    "    if episode % log_interval == 0:\n",
    "        print('Episode {}: start {:.1f}, end {:.1f} because {} @ t={}, last reward {:.1f}, running reward {:.1f}' \\\n",
    "              .format(episode, env.starting_portfolio_value, env.portfolio_value(), msg['msg'],\n",
    "                      env.cur_timestep, reward, running_reward))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 72
    },
    "colab_type": "code",
    "id": "Cvq9Fj0cHhQp",
    "outputId": "f52362d5-13cc-479b-be5f-3853bb9507ca"
   },
   "outputs": [],
   "source": [
    "total_rewards = 0\n",
    "total_profits = 0\n",
    "failed_goes = 0\n",
    "num_goes = 50\n",
    "\n",
    "for j in range(num_goes):\n",
    "    env.reset()\n",
    "    reward_this_go = -1e8\n",
    "    for i in range(0, env.series_length + 1):\n",
    "        action = model.act(env.state)\n",
    "        next_state, reward, done, msg = env.step(action)\n",
    "        if msg['msg'] == 'done':\n",
    "            reward_this_go = env.portfolio_value()\n",
    "            break\n",
    "        if done:\n",
    "            break\n",
    "    total_profits += (env.portfolio_value() - env.starting_portfolio_value) / env.starting_portfolio_value\n",
    "    if reward_this_go == -1e8:\n",
    "        failed_goes += 1\n",
    "    else:\n",
    "        total_rewards += reward_this_go\n",
    "\n",
    "if failed_goes == num_goes:\n",
    "    print('Failed all!')\n",
    "else:\n",
    "    print('Failed goes: {}/{}, Average reward per successful game: {}'.format(\n",
    "        failed_goes, num_goes, total_rewards / (num_goes - failed_goes)))\n",
    "    print('Average profit per game: ', total_profits / num_goes)\n",
    "    print('Average profit per finished game: ', total_profits / (num_goes - failed_goes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ulEYBN9bHhQt"
   },
   "outputs": [],
   "source": [
    "print('Starting portfolio value ', env.portfolio_value())\n",
    "for i in range(0, env.series_length + 1):\n",
    "    action = model.act(env.state)\n",
    "    next_state, reward, done, msg = env.step(action)\n",
    "    if msg['msg'] == 'bankrupted self':\n",
    "        print('Bankrupted self by 1')\n",
    "        break\n",
    "    if msg['msg'] == 'sold more than have':\n",
    "        print('Sold more than have by 1')\n",
    "        break\n",
    "    print('{}, have {} stocks and {} cash'.format(msg['msg'], next_state[0], next_state[1]))\n",
    "    if msg['msg'] == 'done':\n",
    "        print(next_state, reward)\n",
    "        print('Total portfolio value ', env.portfolio_value())\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DZ0TYNXFHhQy"
   },
   "outputs": [],
   "source": [
    "env.reset()\n",
    "complete_game = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VcyNo1ZwHhQ4",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "while not complete_game:\n",
    "    bought_apl_at = []\n",
    "    sold_apl_at = []\n",
    "    bought_apl_at_orig = []\n",
    "    sold_apl_at_orig = []\n",
    "    nothing_at = []\n",
    "    b_action_times = []\n",
    "    s_action_times = []\n",
    "    n_action_times = []\n",
    "    starting_val = env.starting_portfolio_value\n",
    "    print('Starting portfolio value: {}'.format(starting_val))\n",
    "    for i in range(0, env.series_length + 1):\n",
    "        action = model.act(env.state)\n",
    "        if action == 0:\n",
    "            bought_apl_at.append(apl_open[env.cur_timestep])\n",
    "            bought_apl_at_orig.append(apl_open_orig[env.cur_timestep])\n",
    "            b_action_times.append(env.cur_timestep)\n",
    "        if action == 1:\n",
    "            sold_apl_at.append(apl_close[env.cur_timestep])\n",
    "            sold_apl_at_orig.append(apl_close_orig[env.cur_timestep])\n",
    "            s_action_times.append(env.cur_timestep)\n",
    "        if action == 2:\n",
    "            nothing_at.append(0)\n",
    "            n_action_times.append(env.cur_timestep)\n",
    "        next_state, reward, done, msg = env.step(action)\n",
    "        if msg['msg'] == 'bankrupted self':\n",
    "            env.reset()\n",
    "            break\n",
    "        if msg['msg'] == 'sold more than have':\n",
    "            env.reset()\n",
    "            break\n",
    "        if msg['msg'] == 'done':\n",
    "            print('{}, have {} stocks and {} cash'.format(msg['msg'], next_state[0], next_state[1]))\n",
    "            val = env.portfolio_value()\n",
    "            print('Finished portfolio value {}'.format(val))\n",
    "            complete_game = True\n",
    "            env.reset()\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 338
    },
    "colab_type": "code",
    "id": "BKO4l8dQHhQ9",
    "outputId": "0981e554-daed-41bf-d162-fc45ff9cb8ba"
   },
   "outputs": [],
   "source": [
    "plt.figure(1, figsize=(14, 5))\n",
    "plt.plot(range(0, len(apl_open)), apl_open)\n",
    "plt.plot(b_action_times, bought_apl_at, 'ro')\n",
    "plt.plot(s_action_times, sold_apl_at, 'go')\n",
    "plt.plot(n_action_times, nothing_at, 'yx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 338
    },
    "colab_type": "code",
    "id": "5qo30DMeHhRD",
    "outputId": "dc447941-a85f-4e46-be84-a24c09a677f3"
   },
   "outputs": [],
   "source": [
    "plt.figure(1, figsize=(14, 5))\n",
    "plt.plot(range(0, len(apl_open_orig)), apl_open_orig)\n",
    "plt.plot(b_action_times, bought_apl_at_orig, 'ro')\n",
    "plt.plot(s_action_times, sold_apl_at_orig, 'go')\n",
    "plt.plot(n_action_times, nothing_at, 'yx')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "StockTradingAC.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}