DeepSequenceModelingPricePrediction.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gWng0TQrnkIx"
      },
      "source": [
        " <html>\n",
        "<head><center>\n",
        "<h1>Predicting Forex Data Using Recurrent Neural Network Deep Sequence Modeling </b></h1>\n",
        "<br>\n",
        "<img src=\"https://i.ibb.co/jh6qD0t/RNN.png\" alt=\"RNN\" border=\"0\"></a>\n",
        "</center>\n",
        "</head>\n",
        "\n",
        "<html>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SLib0ib6ks-0"
      },
      "source": [
        "### **IMPORTING LIBRARIES**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "0GmRFPkdLhy3",
        "outputId": "88306d57-035b-4306-883d-1889e0b82b2e"
      },
      "outputs": [],
      "source": [
        "# Import Tensorflow 2.0\n",
        "%tensorflow_version 2.x\n",
        "import tensorflow as tf\n",
        "\n",
        "#Install Stock API Client (forex aggregates not currently implemented however I addressed the issue and waiting for them to get back to me. Using requests for now)\n",
        "%pip install --upgrade --force-reinstall polygon-api-client\n",
        "%pip install requests\n",
        "%pip install matplotlib\n",
        "\n",
        "#Import other packages\n",
        "from polygon import RESTClient\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import classification_report\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import requests\n",
        "import os\n",
        "import io\n",
        "import time\n",
        "import random\n",
        "import datetime\n",
        "import functools\n",
        "import csv\n",
        "import glob\n",
        "from google.colab import drive\n",
        "from tqdm import tqdm\n",
        "from IPython import display as ipythondisplay"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-eCBjdb40U2x"
      },
      "source": [
        "# **1 DATA PREPROCESSING AND CLEAN UP**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xFTwf3bizfpb"
      },
      "source": [
        "### **1.1 TWO FUNCTIONS: ONE TO ACCESS POLYGON API, ANOTHER TO CONVERT TIME STAMP FOR A DATA POINT.**\n",
        "From 2010-01 to 2020-12 A total 120 Months.\n",
        "The function access the Polygon API in 1 hour timeframes in a span of a month.\n",
        "Each Monthly Batch comes in as a JSON file which is converted to a pandas DataFrame then converted into a CSV file."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PuUDWFKHzmJO"
      },
      "source": [
        "<html>\n",
        "<head>\n",
        "<img src=\"https://i.ibb.co/XpsFbSH/Polygon.png\" alt=\"Polygon\" border=\"0\"></a>\n",
        "</head>\n",
        "\n",
        "<html>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a5r5fo-Uztez"
      },
      "outputs": [],
      "source": [
        "def ts_to_datetime(ts) -> str:\n",
        "  return datetime.datetime.fromtimestamp(ts / 1000.0).strftime('%Y-%m-%d %H:%M')\n",
        "\n",
        "def print_csv(start, end, year, month):\n",
        "  # API key\n",
        "  key = #PUT API KEY FROM POLYGON HERE#  \n",
        "\n",
        "  endpoint = 'https://api.polygon.io/v2/aggs/ticker/C:GBPJPY/range/1/hour/{dfrom}/{to}?apiKey={key}&limit=50000'\n",
        "  resp = requests.get(endpoint.format(dfrom = start, to = end, key = key ))\n",
        "\n",
        "  my_dictionary = {'t': [' '], 'o': [0.0] , 'h': [0.0], 'l': [0.0], 'c': [0.0], 'v': [0.0], 'vw': [0.0], 'avgOHLC': [0.0]}\n",
        "\n",
        "  for result in resp.json()['results']:\n",
        "    dt = ts_to_datetime(result[\"t\"])\n",
        "    avgOHLC = (result['o'] + result['h'] + result['l'] + result['c']) / 4\n",
        "\n",
        "    my_dictionary['t'].append(dt)             # Time\n",
        "    my_dictionary['o'].append(result['o'])    # Open Price\n",
        "    my_dictionary['h'].append(result['h'])    # High Price\n",
        "    my_dictionary['l'].append(result['l'])    # Low Price\n",
        "    my_dictionary['c'].append(result['c'])    # Close Price\n",
        "    my_dictionary['v'].append(result['v'])    # Volume\n",
        "    my_dictionary['vw'].append(result['vw'])  # Volume Weighted\n",
        "    my_dictionary['avgOHLC'].append(avgOHLC)  # Average Price\n",
        "\n",
        "  # converts dictionary to pandas DataFrame\n",
        "  df = pd.DataFrame(my_dictionary)            \n",
        "  \n",
        "  filename = \"/drive/My Drive/CSV2/\" + year + \"-\" + month + \".csv\"\n",
        "\n",
        "  # converts pandas DataFrame into a csv file\n",
        "  df.to_csv(filename)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yBDiJJ-uzxgJ"
      },
      "source": [
        "### **1.2 SCRIPT FOR AUTOMATING A TIMELINE OF 10 YEARS MONTHLY API CALLS**\n",
        "Resulting in 120 Monthly CSV Files with each file containing 1 hour data points in a span of a month."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 346
        },
        "id": "8gdzj-Jiz25G",
        "outputId": "616d140e-d6b5-436b-fb4a-5bd7194b19fa"
      },
      "outputs": [],
      "source": [
        "# specify training data timeline: 10 years\n",
        "# from 2010 to 2020\n",
        "\n",
        "import datetime\n",
        "import calendar\n",
        "\n",
        "year = 2010\n",
        "month = 1\n",
        "\n",
        "for i in range(0, 11):\n",
        "\n",
        "  for j in range(0, 12):\n",
        "    _, num_days = calendar.monthrange(year + i, month + j)\n",
        "\n",
        "    start_day = datetime.date(year + i, month + j, 1)\n",
        "    end_day = datetime.date(year + i, month + j, num_days)\n",
        "\n",
        "    print_csv(start_day, end_day, str(year + i), str(month + j))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w_AUje66z5KY"
      },
      "source": [
        "### **1.3 CONCATENATING ALL 120 MONTHLY CSV FILES INTO ONE CSV FILE.**\n",
        "The resulting csv file has 68,293 data points from 2010-01 to 2020-12 in 1 hour time frames."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G_5VfUA-z8S8"
      },
      "outputs": [],
      "source": [
        "cwd = os.getcwd()\n",
        "\n",
        "os.chdir('/drive/My Drive/CSV2/')\n",
        "\n",
        "extension = 'csv'\n",
        "all_filenames = [i for i in glob.glob('*.{}'.format(extension))]\n",
        "\n",
        "# combines all files in the list\n",
        "combined_csv = pd.concat([pd.read_csv(f) for f in all_filenames ])\n",
        "# export to csv\n",
        "combined_csv.to_csv( \"forexDataGBPJYP.csv\", index=False, encoding='utf-8-sig')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bwo9CQo_lDUr"
      },
      "source": [
        "### **1.4 UPLOADING CSV FILE**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72,
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCkgewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwogICAgICBwZXJjZW50LnRleHRDb250ZW50ID0KICAgICAgICAgIGAke01hdGgucm91bmQoKHBvc2l0aW9uIC8gZmlsZURhdGEuYnl0ZUxlbmd0aCkgKiAxMDApfSUgZG9uZWA7CiAgICB9CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "ok": true,
              "status": 200,
              "status_text": ""
            }
          }
        },
        "id": "25a0S_PTO2j5",
        "outputId": "c58d9d56-e6da-44e7-8c70-afeaba968c9f"
      },
      "outputs": [],
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fYlRVTKIlJCS"
      },
      "source": [
        "## **1.5 DATA PREPROCESSING FOR MODEL**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "19AloFZfazvk"
      },
      "source": [
        "To start processing, we define a function to convert UNIX timestamps that the Polygon API returns to dates and define the API key as well as all vectors we will use for data collection and preprocessing."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ubdSqdNlaYVM"
      },
      "outputs": [],
      "source": [
        "#timestamp to datetime converter as polygon returns timestamps\n",
        "def ts_to_datetime(ts) -> str:\n",
        "  return datetime.datetime.fromtimestamp(ts / 1000.0).strftime('%Y-%m-%d %H:%M')\n",
        "#Define the key for API and initialize all Vectors to be used for processing\n",
        "key_ = #PUT API KEY FROM POLYGON HERE#    \n",
        "dateVector = []\n",
        "ohlcVector = []"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xeMv5iTueGJ5"
      },
      "source": [
        "### **1.6 DATA SELECTION**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pGIWr7QlbRge"
      },
      "source": [
        "#### **1.7 OPTIONAL CELL TO FETCH DATA FROM API INSTEAD OF DATABASE**\n",
        "Run this cell to populate the ohlcVector with average OHLC (open, high, low,  close) prices  obtained from the API. This works the best with daily data due to the way the Polygon API aggregates prices and allows for the flexibility of choosing any currency pair "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uIshUZR0abPB"
      },
      "outputs": [],
      "source": [
        "from_ = \"2009-10-01\"\n",
        "to_ = \"2020-05-01\"\n",
        "endpoint = 'https://api.polygon.io/v2/aggs/ticker/C:USDJPY/range/1/day/{dfrom}/{to}?unadjusted=true&sort=asc&apiKey={key}&limit=50000'\n",
        "resp = requests.get(endpoint.format(dfrom = from_, to = to_, key = key_ ))\n",
        "for result in resp.json()['results']:  \n",
        "    dt = ts_to_datetime(result[\"t\"])\n",
        "    avgOHLC = (result['o'] + result['h'] + result['l'] + result['c']) / 4\n",
        "    dateVector.append(dt[5:])\n",
        "    ohlcVector.append(float(avgOHLC))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WKLxlF9Ic9B4"
      },
      "source": [
        "#### **1.8.1 CELL TO FETCH DATA FROM UPLOADED FILE OR DATABASE**\n",
        "If the cell above was not run, run this cell to use data from an uploaded csv file or database. This was necessary as we had to write a script to concatenate all 4 hour data and 1 hour data for a given currency from the API due to restrictions with how data is aggregated on the API side."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        },
        "id": "jeoksN1DdEzN",
        "outputId": "19ceae10-c8cd-4722-c1f4-6ebb6d5fac59"
      },
      "outputs": [],
      "source": [
        "#Create price difference vector to be used as input\n",
        "df2 = pd.read_csv(io.BytesIO(uploaded['FOREXdataGBP-JYP.csv'])) \n",
        "ohlcVector = df2['avgOHLC'].tolist()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zZKzRVEHaDAh"
      },
      "source": [
        "### **1.8.2 FETCH DATA FROM MONGODB MLAB DATABASE**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JdrK-oapZ9Hd"
      },
      "outputs": [],
      "source": [
        "\n",
        "from pymongo import MongoClient\n",
        "from pprint import pprint \n",
        "\n",
        "mongo_client = MongoClient(#MONGO CREDENTIALS HERE IF APPLICABLE#)\n",
        "\n",
        "# database name\n",
        "db = mongo_client[\"DataMining2021\"]\n",
        "# collection name\n",
        "col = db[\"4HRGBPYEN\"]\n",
        "\n",
        "result = col.find()\n",
        "\n",
        "openValue = []\n",
        "highValue = []\n",
        "lowValue = []\n",
        "closeValue = []\n",
        "avgOHLCValue = []\n",
        "for doc in col.find():\n",
        "    openValue += [doc[\"1\"]]\n",
        "    highValue += [doc[\"2\"]]\n",
        "    lowValue += [doc[\"3\"]]\n",
        "    closeValue += [doc[\"4\"]]\n",
        "    avgOHLCValue += [doc[\"7\"]]\n",
        "\n",
        "print(\"avgOHLCValue: \", closeValue)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N8Kmis8ZgmRd"
      },
      "source": [
        "# **2 PRICE PREDICTION OVER TIME MODEL**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NKE5hqmMeP3H"
      },
      "source": [
        "### **2.1 GENERATING CHANGES IN PRICE AND SPLITTING DATA FOR TRAINING AND TESTING**\n",
        "Instead of using prices as inputs to our model, which are sensitive to trends due to time, we use percent changes in price which do not change as prices go up or down. This also helps to provide bounds to the data. For our splitting of the data, we use a standard 80/20 split between training and testing data.  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JSd3I3peT5Ty",
        "outputId": "471bdd58-92cb-4db6-e884-05f4ec981aa7"
      },
      "outputs": [],
      "source": [
        "dpriceVector = []\n",
        "for i in range(0,len(ohlcVector)-1):\n",
        "  dpriceVector.append(float('{:.7f}'.format(((ohlcVector[i+1]-ohlcVector[i])/ohlcVector[i])))*100)\n",
        "vocab_size = len((set(dpriceVector)))\n",
        "print(\"TOTAL SIZE:\", len(dpriceVector))\n",
        "print(\"There are\", vocab_size, \"unique characters in the dataset\")\n",
        "split_point = int(len(dpriceVector)*0.8)\n",
        "trainingData = dpriceVector[:split_point]    \n",
        "testData = dpriceVector[split_point:]\n",
        "trainingData_len = len(trainingData)\n",
        "testData_len = len(testData)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VZgRkoSogHpB"
      },
      "source": [
        "### **2.3 SHAPING DATA FOR MODEL**\n",
        "The idea for our model is that it takes a sequence of percent changes in price and tries to predict the next percent change. Therefore, the input for our model needs to be a list of percent change sequences and our output will be a list of what the next percent change should be. Since we need to define the sequence length, we will define this function now and use it later when we define hyperparameters."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QzJ656Cve7Dx"
      },
      "outputs": [],
      "source": [
        "def createTrainingBatch(seq_length):\n",
        "  x_train = []\n",
        "  y_train = []\n",
        "  for x in range(seq_length, trainingData_len):\n",
        "    x_train.append(trainingData[x-seq_length:x])\n",
        "    y_train.append(trainingData[x])\n",
        "  x_train = np.array(x_train)\n",
        "  y_train = np.array(y_train)\n",
        "  x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))\n",
        "  return x_train, y_train"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZbWvcQLLlU14"
      },
      "source": [
        "## **2.4 BUILDING THE MODEL**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mY6Qzoz4ML1U"
      },
      "source": [
        "<html>\n",
        "<head>\n",
        "<h1>Recurrrent Neural Network using Long Short Term Memory (LSTM) </b></h1>\n",
        "<br>\n",
        "<img src=\"https://i.ibb.co/y5FDfpW/LSTM.png\" alt=\"LSTM\" border=\"0\">\n",
        "<img src=\"https://i.ibb.co/v1cFZch/LSTM5.png\" alt=\"LSTM5\" border=\"0\">\n",
        "<img src=\"https://i.ibb.co/6bMHPN2/Many-To-One.png\" alt=\"Many-To-One\" border=\"0\">\n",
        "</head>\n",
        "\n",
        "<html>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WUTe7ov6iVHy"
      },
      "source": [
        "### **2.4.1 MODEL ARCHITECTURE**\n",
        "This function defines the architecture our model uses. We stack 3 LSTMs to provide layers of abstraction that allow the model to create its own intermediary data from which it will generate a prediction through the final dense layer with 1 output. We placed dropout layers between each LSTM layer to help prevent overfitting. The Model uses 1 dense layer."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bLXAVM9U5PxZ"
      },
      "outputs": [],
      "source": [
        "def build_model(rnn_units, input_shape_):\n",
        "  model = tf.keras.Sequential([\n",
        "    tf.keras.layers.LSTM(rnn_units, return_sequences=True, input_shape=input_shape_),\n",
        "    tf.keras.layers.Dropout(0.2),\n",
        "    tf.keras.layers.LSTM(rnn_units, return_sequences=True),\n",
        "    tf.keras.layers.Dropout(0.2),\n",
        "    tf.keras.layers.LSTM(rnn_units),\n",
        "    tf.keras.layers.Dense(1)\n",
        "  ])\n",
        "  return model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sN6x535SmIAh"
      },
      "source": [
        "### **2.4.2 SUMMARY OF A SAMPLE MODEL**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5lof9i9yTdet",
        "outputId": "019483c9-5fbd-4d56-ae9c-8c7cc80a5bdb"
      },
      "outputs": [],
      "source": [
        "model = build_model(256, (250, 1))\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zmmcw4nLlv9p"
      },
      "source": [
        "### **2.4.3 DEFINING HYPERPARAMETERS**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CX8FIvkCHukz"
      },
      "outputs": [],
      "source": [
        "# Optimization parameters:\n",
        "num_epochs = 120\n",
        "batch_size_ = 32  \n",
        "seq_length = 125 \n",
        "learning_rate = 1e-5 \n",
        "\n",
        "# Model parameters: \n",
        "rnn_units = 150  # Experiment between 1 and 2048\n",
        "\n",
        "# Checkpoint location: \n",
        "checkpoint_dir = './training_checkpoints'\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, \"my_ckpt\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ihn3u5ajl4CS"
      },
      "source": [
        "### **2.4.4 TRAINING THE MODEL**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1f5V_0DZN7zI"
      },
      "source": [
        "<html>\n",
        "<head>\n",
        "<h1>Training the Model: Backpropagation Through Time </b></h1>\n",
        "\n",
        "\n",
        "<br>\n",
        "<img src=\"https://i.ibb.co/LppKzzW/RNNBack-Prop.png\" alt=\"RNNBack-Prop\" border=\"0\"></a>\n",
        "</head>\n",
        "\n",
        "<html>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XQaLcBjnIKJM",
        "outputId": "4f5a8279-1083-4b8f-a5b7-4a3b4f584301"
      },
      "outputs": [],
      "source": [
        "#generate the training batch \n",
        "x_train,y_train = createTrainingBatch(seq_length)\n",
        "#build the model\n",
        "model = build_model(rnn_units, (x_train.shape[1],1))\n",
        "#define optimizer and loss function\n",
        "model.compile(optimizer='Adam', loss='mean_squared_error')\n",
        "#fit the model to the data\n",
        "history = model.fit(x_train, y_train, epochs=num_epochs, batch_size=batch_size_)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SG6Fp_CDlA7H"
      },
      "source": [
        "### **2.4.8 PLOTTING THE LOSS THROUGHOUT TRAINING**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "dTuVv3KqlGM8",
        "outputId": "57152991-ef95-44ec-ebb6-f49af8500b33"
      },
      "outputs": [],
      "source": [
        "plt.plot(history.history['loss'], label='train')\n",
        "plt.legend()\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fVNj5RhClZOW"
      },
      "source": [
        "# **3 TESTING THE MODEL**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G488IvsOlsVw"
      },
      "source": [
        "### **3.1 TEST DATA GENERATION**\n",
        "Since LSTMs rely on developing a hidden state from seeing earlier data, we will have to concatenate the testing data to the training data to see how well the model is able to continue predicting the market"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UBdZRcd-lra4"
      },
      "outputs": [],
      "source": [
        "generation_length = 10\n",
        "x_test = []\n",
        "for x in range(seq_length,len(dpriceVector[0:-(generation_length)])):\n",
        "  x_test.append(dpriceVector[x-seq_length:x])\n",
        "x_test = np.array(x_test)\n",
        "x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dz-D1fn_nUCr"
      },
      "source": [
        "### **3.2 PRICE PREDICTION**\n",
        "One problem we saw with many model online is how they handle prediction. Since the goal is predicting prices into the future, the model should be making prediction based off its own predictions. Many online models simply run the model on the test data which does not accomplish this."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cFyc3XTWPtAY",
        "outputId": "cc8d5e24-3f91-41bd-ecdf-dd1b4ba4f2e3"
      },
      "outputs": [],
      "source": [
        "predicted_price = []\n",
        "for i in range(0,generation_length):\n",
        "  predicted_price = model.predict(x_test)\n",
        "  print(predicted_price[-1][0])\n",
        "  add_price = float('{:.6f}'.format(predicted_price[-1][0]))\n",
        "  x_test_copy = x_test.copy()\n",
        "  np.squeeze(x_test_copy)\n",
        "  x_test_copy = np.append(x_test_copy,add_price)\n",
        "  new_data_sequence = x_test_copy[(0-seq_length):]\n",
        "  new_data_sequence = new_data_sequence.reshape(1,seq_length,1)\n",
        "  x_test = np.concatenate((x_test,new_data_sequence))\n",
        "predicted_prices = [predicted_price[i] for i in range(0,len(predicted_price))]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CE_1_DJtn14Z"
      },
      "source": [
        "### **3.3 CONVERTING PERCENT PRICE CHANGES BACK TO ACTUAL PRICES**\n",
        "Since our model outputs percent changes in price, we must convert this back to actual prices given a starting price. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vPd7o1F0oAKs"
      },
      "outputs": [],
      "source": [
        "calculatedOriginalPrices = []\n",
        "calculatedOriginalPrices.append(ohlcVector[seq_length])\n",
        "for i in range(seq_length,len(dpriceVector)):\n",
        "  nextPriceOriginal = calculatedOriginalPrices[i-seq_length] * ((dpriceVector[i])/100+1)\n",
        "  calculatedOriginalPrices.append(nextPriceOriginal)\n",
        "calculatedPredictedPrices = []\n",
        "calculatedPredictedPrices.append(ohlcVector[seq_length])  \n",
        "for i in range(0,len(predicted_prices)):\n",
        "  nextPricePredicted = calculatedPredictedPrices[i] * ((predicted_prices[i])/100+1)\n",
        "  calculatedPredictedPrices.append(nextPricePredicted)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YsPMA-mcoOrm"
      },
      "source": [
        "### **3.4 VISUALIZING THE RESULTS**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 627
        },
        "id": "bz8yvcfJRDME",
        "outputId": "a8df0ee2-a017-4984-ce19-0a303527d4a9"
      },
      "outputs": [],
      "source": [
        "plt.plot(dpriceVector[seq_length:], color='black', label=\"Actual price movement\")\n",
        "plt.plot(predicted_prices, color='red', label=\"Predicted price movement\")\n",
        "plt.title(\"Price change\")\n",
        "plt.xlabel('time')\n",
        "plt.ylabel(\"Price Change\")\n",
        "plt.legend()\n",
        "plt.show()\n",
        "plt.plot(calculatedOriginalPrices, color='black', label=\"Actual prices\")\n",
        "plt.plot(calculatedPredictedPrices, color='red', label=\"Predicted prices\")\n",
        "plt.title(\"Price\")\n",
        "plt.xlabel('time')\n",
        "plt.ylabel(\"Price\")\n",
        "plt.legend()\n",
        "plt.show()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CgJp-FAvoYsE"
      },
      "source": [
        "# **4 SHORT TERM CATEGORICAL PRICE PREDICTION MODEL**\n",
        "After experimenting with the price prediction over time model, we came to the conclusion that predicting prices over the long term is not feasible given the amount of external variables at play that compound over time. Instead, we decided to pursue a model that could be used as more of a trading tool. Based on previous prices, we try to predict whether the price will be moving up or down in the next time frame and how big will the movement be. Because the output is categorical, the model will also be able to provide confidence metric which we could leverage to find optimal times to enter the market."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1F5EkBwopZvk"
      },
      "source": [
        "### **4.1 VISUALIZING TRAINING DATA THROUGH A HISTOGRAM**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 268
        },
        "id": "7l1uUfaqpnX0",
        "outputId": "ddc76d09-bdb3-409d-df2f-99180e3ee297"
      },
      "outputs": [],
      "source": [
        "plt.hist(trainingData, edgecolor=\"black\",bins=100)\n",
        "plt.xlim(-0.5,0.5)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W6yuPbqopr5m"
      },
      "source": [
        "## **4.2 DATA PREPROCESSING**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pJgC08Dzqy8T"
      },
      "source": [
        "### **4.3 VECTORIZING OUTPUTS AND SHAPING DATA FOR MODEL**\n",
        "Since we will be predicting categories of price movement, we must define what is considered a big, medium, and small movement. We first accomplished this by simply visualizing the data through a histogram and picking the appropriate cutoffs. These are the values we settled on for 4-hour data.\n",
        "\n",
        "0.  -> Big Move Down (:-2]\n",
        "1.  -> Medium Move Down (-2:-0.75]\n",
        "2.  ->Small Move Down (-0.75:0)\n",
        "3.  ->Small Move Up (0:0.75) \n",
        "4.  ->Medium Move Up [0.75:2)\n",
        "5.  ->Big Move Up [0.2:)\n",
        "6.  ->No movement 0.0\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OIGjKHhm2qB4"
      },
      "outputs": [],
      "source": [
        "def createCategoricalTrainingBatch(seq_length):\n",
        "  x_train = []\n",
        "  y_train = []\n",
        "  #print(np.average(trainingData))\n",
        "  high_cutoff = 0.97*np.std(trainingData)\n",
        "  medium_cutoff = 0.35*np.std(trainingData)\n",
        "  for x in range(seq_length, trainingData_len):\n",
        "    x_train.append(trainingData[x-seq_length:x])\n",
        "    if trainingData[x] <= -(high_cutoff):\n",
        "      y_train.append(0)\n",
        "    elif trainingData[x] <= -(medium_cutoff) and trainingData[x] > -high_cutoff:\n",
        "      y_train.append(1)\n",
        "    elif trainingData[x] < 0 and trainingData[x] > -(medium_cutoff):\n",
        "      y_train.append(2)\n",
        "    elif trainingData[x] > 0 and trainingData[x] < medium_cutoff:\n",
        "      y_train.append(3)\n",
        "    elif trainingData[x] >= medium_cutoff and trainingData[x] < high_cutoff:\n",
        "      y_train.append(4)\n",
        "    elif trainingData[x] >= high_cutoff:\n",
        "      y_train.append(5)\n",
        "    else:\n",
        "      y_train.append(6)\n",
        "\n",
        "  x_train = np.array(x_train)\n",
        "  y_train = np.array(y_train)\n",
        "  x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))\n",
        "  return x_train, y_train;"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4JBWRTaVsQCC"
      },
      "source": [
        "# **5 BUILDING THE MODEL**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GDaUQzWSsXzS"
      },
      "source": [
        "### **5.1 MODEL ARCHITECTURE**\n",
        "Our model for the cateogrical predictions is very similar to our initial model for price predictions over time. The big exception is that the output dense layer now has 7 outputs for each of the categories we defined."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZFmvZvLooKiy"
      },
      "source": [
        "<html>\n",
        "<head>\n",
        "<h1>Recurrrent Neural Network using Categorical Data </b></h1>\n",
        "<br>\n",
        "<img src=\"https://i.ibb.co/y5FDfpW/LSTM.png\" alt=\"LSTM\" border=\"0\">\n",
        "<img src=\"https://i.ibb.co/v1cFZch/LSTM5.png\" alt=\"LSTM5\" border=\"0\">\n",
        "<img src=\"https://i.ibb.co/xFjgXk0/Many-To-Many.png\" alt=\"Many-To-Many\" border=\"0\">\n",
        "</head>\n",
        "\n",
        "<html>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zRzSODowFVhU"
      },
      "outputs": [],
      "source": [
        "#CHANGED MODEL NEEDS TO HAVE A DENSE LAYER EQUAL TO NUMBER OF CATEGORIES WE MADE\n",
        "def build_model(rnn_units, seq_length):\n",
        "  model = tf.keras.Sequential([\n",
        "    tf.keras.layers.LSTM(rnn_units, return_sequences=True, input_shape=(seq_length,1)),\n",
        "    tf.keras.layers.Dropout(0.5),\n",
        "    tf.keras.layers.LSTM(rnn_units, return_sequences=True),\n",
        "    tf.keras.layers.Dropout(0.5),\n",
        "    tf.keras.layers.LSTM(rnn_units, return_sequences=True),\n",
        "    tf.keras.layers.Dropout(0.5),\n",
        "    tf.keras.layers.LSTM(rnn_units),\n",
        "    tf.keras.layers.Dense(7)\n",
        "  ])\n",
        "  return model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9Xw8BekU0nKF"
      },
      "source": [
        "###Categories:\n",
        "* Big Move Down\n",
        "* Medium Move Down\n",
        "* Small Move Down\n",
        "* No Movement\n",
        "* Small Move Up\n",
        "* Medium Move Up\n",
        "* Big Move Up\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j1dpy5q7slUU"
      },
      "source": [
        "### **5.2 DEFINING HYPERPARAMTERS**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CzT1iUv9Fl-6"
      },
      "outputs": [],
      "source": [
        "# Optimization parameters:\n",
        "num_epochs_categorical = 300  \n",
        "batch_size_categorical = 32  \n",
        "seq_length_categorical = 900  \n",
        "learning_rate_categorical = 1e-5 \n",
        "\n",
        "# Model parameters: \n",
        "rnn_units_categorical = 512 \n",
        "\n",
        "# Checkpoint location: \n",
        "checkpoint_dir_categorical = './training_checkpoints_categorical'\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir_categorical, \"my_ckpt\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cOL9llHis6QU"
      },
      "source": [
        "### **5.3 GENERATING THE CATEGORICAL TRAINING BATCH**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P4Lv-PkYtIQG",
        "outputId": "d41be7b8-e5bc-4194-d685-39e909207a3c"
      },
      "outputs": [],
      "source": [
        "x_train, y_train = createCategoricalTrainingBatch(seq_length_categorical)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9-dbqOT6tOe1"
      },
      "source": [
        "### **5.4 VISUALIZING CATEGORY DISTRIBUTION**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "SFXPKWzKr5Mf",
        "outputId": "bc5b28a6-85e6-4141-a27e-459322f20295"
      },
      "outputs": [],
      "source": [
        "plt.hist(y_train, edgecolor=\"black\",bins=20)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WG3JpXICtbzo"
      },
      "source": [
        "### **5.5 TRAINING THE MODEL**\n",
        "We train the model in much the same way as we trained the price prediction over time model except that we are using the Adamax optimizer which is more suited for categorical data and we are using Sparse Categorical Cross Entropy as the loss function (ensuring it applies a softmax over outputs before calculations are done with from_logits=True). "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qr2rPt4UFthU",
        "outputId": "3f45587c-c4ce-4ef5-a85b-2102c3f65a99"
      },
      "outputs": [],
      "source": [
        "model = build_model(rnn_units_categorical, seq_length_categorical)\n",
        "model.compile(optimizer='Adamax', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True))\n",
        "history = model.fit(x_train, y_train, epochs=num_epochs_categorical, batch_size=batch_size_categorical)\n",
        "model.save_weights(checkpoint_prefix)\n",
        "files.download('/content/training_checkpoints_categorical/*')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "grSPuqbwt-3l"
      },
      "source": [
        "### **5.6 PLOTTING THE LOSS THROUGHOUT TRAINING**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "N-tUD48Zsvg2",
        "outputId": "4a70c3f0-eaca-467c-999a-91ebb108680c"
      },
      "outputs": [],
      "source": [
        "plt.plot(history.history['loss'], label='train')\n",
        "plt.legend()\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VEoLJ0kh3ZS9"
      },
      "source": [
        "### **5.7 Optional Building Model from Saved Weights**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        },
        "id": "Ruh72tgPmSa0",
        "outputId": "624569ff-fa21-4c4b-ba02-41ec8f0e1a4f"
      },
      "outputs": [],
      "source": [
        "model = build_model(rnn_units_categorical, seq_length_categorical)\n",
        "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir_categorical))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aRiMVY6SuYci"
      },
      "source": [
        "# **6 TESTING THE MODEL**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZduMwQ8o3Ouq"
      },
      "source": [
        "### **6.1 SIMPLE SINGLE DATA CHECKING**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V93NWk0Qi9tr"
      },
      "source": [
        "<html>\n",
        "<head>\n",
        "<h1>SoftMax Activation Function on Categorical Ouput Data</b></h1>\n",
        "\n",
        "<img src=\"https://i.ibb.co/HhM6t8z/softmax.webp\" alt=\"softmax\" border=\"0\">\n",
        "<img src=\"https://i.ibb.co/Fs8vKnS/softmax-activation-function.jpg\" alt=\"softmax-activation-function\" border=\"0\">\n",
        "</head>\n",
        "\n",
        "<html>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oM1Y9GqAIift",
        "outputId": "36a8c216-a998-45a3-d237-ce05c15d5479"
      },
      "outputs": [],
      "source": [
        "x_test = []\n",
        "comparison_point = -5356\n",
        "for x in range(seq_length_categorical,len(dpriceVector[0:comparison_point])):\n",
        "  x_test.append(dpriceVector[x-seq_length_categorical:x])\n",
        "x_test = np.array(x_test)\n",
        "x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))\n",
        "predicted_price_movements = model.predict(x_test)\n",
        "softmax_results = tf.nn.softmax(predicted_price_movements)\n",
        "print(softmax_results[-1])\n",
        "predicted_move = tf.random.categorical(predicted_price_movements, num_samples=1)[-1,0].numpy()\n",
        "print(predicted_move)\n",
        "print(dpriceVector[comparison_point - 1])\n",
        "# FOR REFERENCE\n",
        "#0 - Big Move Down (:-2]\n",
        "#1 - Medium Move Down (-2:-0.75]\n",
        "#2 - Small Move Down (-0.75:0)\n",
        "#3 - Small Move Up (0:0.75) \n",
        "#4 - Medium Move Up [0.75:2)\n",
        "#5 - Big Move Up [2:)\n",
        "#6 - No Movement"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1--VBB_05gqV"
      },
      "source": [
        "### **6.2 ACCURACY SCRIPT - LOOPING THROUGH TEST DATA**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HhGR__tGot1T"
      },
      "outputs": [],
      "source": [
        "x_test = []\n",
        "y_pred = []\n",
        "y_true = []\n",
        "high_cutoff = 0.97*np.std(trainingData)\n",
        "medium_cutoff = 0.35*np.std(trainingData)\n",
        "for x in range(seq_length_categorical,len(dpriceVector)):\n",
        "  x_test.append(dpriceVector[x-seq_length_categorical:x])\n",
        "for x in range(0, testData_len):\n",
        "  if testData[x] <= -high_cutoff:\n",
        "    y_true.append(0)\n",
        "  elif testData[x] <= -medium_cutoff and testData[x] > -high_cutoff:\n",
        "    y_true.append(1)\n",
        "  elif testData[x] < 0 and testData[x] > -medium_cutoff:\n",
        "    y_true.append(2)\n",
        "  elif testData[x] > 0 and testData[x] < medium_cutoff:\n",
        "    y_true.append(3)\n",
        "  elif testData[x] >= medium_cutoff and testData[x] < high_cutoff:\n",
        "    y_true.append(4)\n",
        "  elif testData[x] >= high_cutoff:\n",
        "    y_true.append(5)\n",
        "  else:\n",
        "    y_true.append(6)\n",
        "x_test = np.array(x_test)\n",
        "x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))\n",
        "predicted_price_movements = model.predict(x_test)\n",
        "raw_test_predictions = predicted_price_movements[(split_point-seq_length_categorical):]\n",
        "y_pred = [tf.random.categorical([predicted_category], num_samples=1).numpy() for predicted_category in raw_test_predictions]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mvbnn12Y1RFs"
      },
      "source": [
        "### **6.3 CONFIDENCE PREDICTIONS**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UX66xzvR3TSG"
      },
      "source": [
        "To utilize the confidence metric produced by the model for each prediction, we run a script to build a predicted array of predictions with >90% confidence and see if has better accuracy than trying to predict the next price at every price point. To this effect, it would be much more useful as a trading tool that could be used alongside other indicators to predict where the market is heading."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d5P_xxO51bmE"
      },
      "outputs": [],
      "source": [
        "x_test = []\n",
        "y_pred = []\n",
        "y_true_full = []\n",
        "y_true = []\n",
        "high_cutoff = 0.97*np.std(trainingData)\n",
        "medium_cutoff = 0.35*np.std(trainingData)\n",
        "for x in range(seq_length_categorical,len(dpriceVector)):\n",
        "  x_test.append(dpriceVector[x-seq_length_categorical:x])\n",
        "for x in range(0, testData_len):\n",
        "  if testData[x] <= -high_cutoff:\n",
        "    y_true_full.append(0)\n",
        "  elif testData[x] <= -medium_cutoff and testData[x] > -high_cutoff:\n",
        "    y_true_full.append(1)\n",
        "  elif testData[x] < 0 and testData[x] > -medium_cutoff:\n",
        "    y_true_full.append(2)\n",
        "  elif testData[x] > 0 and testData[x] < medium_cutoff:\n",
        "    y_true_full.append(3)\n",
        "  elif testData[x] >= medium_cutoff and testData[x] < high_cutoff:\n",
        "    y_true_full.append(4)\n",
        "  elif testData[x] >= high_cutoff:\n",
        "    y_true_full.append(5)\n",
        "  else:\n",
        "    y_true_full.append(6)\n",
        "x_test = np.array(x_test)\n",
        "x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))\n",
        "predicted_price_movements = model.predict(x_test)\n",
        "raw_test_predictions = predicted_price_movements[(split_point-seq_length_categorical):]\n",
        "softmaxed_predictions = tf.nn.softmax(raw_test_predictions)\n",
        "for i in range(0,len(softmaxed_predictions)):\n",
        "  if (np.amax(softmaxed_predictions[i]) > 0.90):\n",
        "    y_pred.append(tf.random.categorical([softmaxed_predictions[i]], num_samples=1).numpy())\n",
        "    y_true.append(y_true_full[i])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L7U27WA85tNU"
      },
      "source": [
        "### **6.3 MODEL ACCURACY**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SsfCAQf0P-ZW",
        "outputId": "3ae2ac16-814b-473a-ff49-4bc76cdaba2d"
      },
      "outputs": [],
      "source": [
        "#MODEL ACCURACY\n",
        "num_correct = 0;\n",
        "for i in range(0, len(y_pred)):\n",
        "  if y_pred[i] == y_true[i]:\n",
        "    num_correct +=1 \n",
        "\n",
        "print(\"TOTAL CORRECT:\",num_correct)\n",
        "print(\"TOTAL DATA:\", len(y_pred))\n",
        "print(\"Accuracy:\", num_correct/len(y_pred))\n",
        "\n",
        "#RANDOM CHOICE ACCURACY\n",
        "num_correct = 0;\n",
        "for i in range(0,len(y_pred)):\n",
        "  if y_true[i] == random.randint(0,6):\n",
        "    num_correct += 1\n",
        "print(\"TOTAL CORRECT:\",num_correct)\n",
        "print(\"TOTAL DATA:\", len(y_pred))\n",
        "print(\"Accuracy:\", num_correct/len(y_pred))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bHeNniPuxs8Q"
      },
      "source": [
        "### **6.4 CONFUSION MATRIX AND OTHER METRICS**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 561
        },
        "id": "9p_qTNW1308U",
        "outputId": "44e9a921-5373-4236-b8f9-a3b44f596692"
      },
      "outputs": [],
      "source": [
        "cf_matrix = confusion_matrix(y_true, np.squeeze(y_pred))\n",
        "sns.heatmap(cf_matrix, annot=True)\n",
        "print(classification_report(y_true, np.squeeze(y_pred)))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "xFTwf3bizfpb",
        "yBDiJJ-uzxgJ",
        "w_AUje66z5KY",
        "bwo9CQo_lDUr",
        "VZgRkoSogHpB",
        "WUTe7ov6iVHy",
        "sN6x535SmIAh",
        "zmmcw4nLlv9p",
        "Ihn3u5ajl4CS",
        "SG6Fp_CDlA7H",
        "fVNj5RhClZOW",
        "G488IvsOlsVw",
        "Dz-D1fn_nUCr",
        "CE_1_DJtn14Z",
        "YsPMA-mcoOrm"
      ],
      "name": "DataMiningFinalProject.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}