TradeBotProject_TensorFlow-Sentiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fdd86d63-de7d-4fee-a9a7-1809fd8cc023",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "549364f0-d609-4e67-ad67-bcc8b976cec4",
   "metadata": {},
   "source": [
    "Using historic data\n",
    "using public sentiment - sentiment analysis\n",
    "using XGBoost model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4aff40f7-2844-4545-b250-3e41788dab33",
   "metadata": {},
   "source": [
    "!pip install tensorflow-metal\n",
    "%env TF_METAL_ENABLED=1\n",
    "!pip install --upgrade tensorflow-metal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2eedd1b3-a483-46b6-8c23-6c0b4da8077b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "\n",
    "# Imports\n",
    "import sys\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "# Enable logging of device placement for debugging purposes\n",
    "# tf.debugging.set_log_device_placement(True)\n",
    "\n",
    "import os\n",
    "# Set the 'TF_METAL_DEVICE_ONLY' environment variable to '1'\n",
    "# This forces TensorFlow to use the Metal device for computation\n",
    "os.environ['TF_METAL_DEVICE_ONLY'] = '1'\n",
    "\n",
    "import platform\n",
    "from os.path import exists\n",
    "from watermark import watermark\n",
    "\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "73f1231e-2333-4b00-831c-bd346565b371",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# List all physical devices TensorFlow can detect on the system\n",
    "devices = tf.config.list_physical_devices()\n",
    "\n",
    "# Print the list of detected physical devices\n",
    "print(devices)\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dc9cd819-3cfc-4627-b746-f6906c994730",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python Platform: macOS-13.0.1-arm64-arm-64bit\n",
      "Python 3.10.10 | packaged by conda-forge | (main, Mar 24 2023, 20:12:31) [Clang 14.0.6 ]\n",
      "Last updated: 2023-05-18T20:35:19.950762-05:00\n",
      "\n",
      "Python implementation: CPython\n",
      "Python version       : 3.10.10\n",
      "IPython version      : 8.13.2\n",
      "\n",
      "Compiler    : Clang 14.0.6 \n",
      "OS          : Darwin\n",
      "Release     : 22.1.0\n",
      "Machine     : arm64\n",
      "Processor   : arm\n",
      "CPU cores   : 10\n",
      "Architecture: 64bit\n",
      "\n",
      "tensorflow: 2.12.0\n",
      "pandas    : 2.0.1\n",
      "sys       : 3.10.10 | packaged by conda-forge | (main, Mar 24 2023, 20:12:31) [Clang 14.0.6 ]\n",
      "platform  : 1.0.8\n",
      "\n",
      "TensorFlow version: 2.12.0\n",
      "Is MPS available?: False\n",
      "Using device: /device:CPU:0\n",
      "Metal device set to: Apple M1 Max\n"
     ]
    }
   ],
   "source": [
    "# Report Technologies\n",
    "print(f'Python Platform: {platform.platform()}')\n",
    "print(f'Python {sys.version}')\n",
    "print(watermark())\n",
    "print(watermark(iversions=True, globals_=globals()))\n",
    "\n",
    "mps_available = tf.config.list_logical_devices(\"MPS\")\n",
    "device = \"/device:MPS:0\" if len(mps_available) > 0 else \"/device:CPU:0\"\n",
    "print(f\"TensorFlow version: {tf.__version__}\\nIs MPS available?: {len(mps_available) > 0}\\nUsing device: {device}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ca4ed731-956e-4db8-89f7-138cae2668df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n    Downloading wikipedia edits for BTC  '"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using wikipedia BTC page edit history as proxy for public interest and sentiment\n",
    "# - additional potential sources for 'sentiment analysis' => tweets, google trends\n",
    "\"\"\"\n",
    "    Downloading wikipedia edits for BTC  \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b8b66fe1-7b9c-4347-8393-15892ca8ac26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 49.9 ms, sys: 12.6 ms, total: 62.5 ms\n",
      "Wall time: 1.02 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#\n",
    "#  BLOCK ONE\n",
    "#\n",
    "\n",
    "import mwclient  # Module for interacting with MediaWiki API.\n",
    "import time  # Module for handling time-related operations.\n",
    "\n",
    "# Initialize a MediaWiki client site which is a class that enables work with a specific wiki site,\n",
    "# in this case, the English (en) version of Wikipedia.\n",
    "site = mwclient.Site('en.wikipedia.org')  # Replace 'en' with the appropriate language code for other sites.\n",
    "\n",
    "# Specify which page to use by creating a Page object using the page's title.\n",
    "page = site.pages['SPDR Gold Shares']  # Replace 'Bitcoin' with the title of alternatively desired Wikipedia page.\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "602583b1-8f15-42d7-a010-21086d5dea79",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 28.7 ms, sys: 7.05 ms, total: 35.7 ms\n",
      "Wall time: 1.06 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#\n",
    "#  BLOCK TWO\n",
    "#\n",
    "\n",
    "# Obtain from Wikipedia the list of revisions for the specified page.\n",
    "revs = list(page.revisions())  # Returns a list of dictionaries containing information about each revision of the page.\n",
    "                              # Each dictionary contains keys such as 'user', 'comment', 'timestamp', etc. that provide\n",
    "                              # details about the revision. The list is sorted in reverse chronological order by default.\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3274d97f-9012-432f-a7d4-d059b5f6851f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('revid', 1151901392),\n",
       "             ('parentid', 1095085650),\n",
       "             ('user', '154.178.207.49'),\n",
       "             ('anon', ''),\n",
       "             ('timestamp',\n",
       "              time.struct_time(tm_year=2023, tm_mon=4, tm_mday=26, tm_hour=23, tm_min=19, tm_sec=43, tm_wday=2, tm_yday=116, tm_isdst=-1)),\n",
       "             ('comment', '/* External links */')])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Look at the first set of revisions for the specified page.\n",
    "# Outputs an ordered dictionary, which is a Python class that functions as a hybrid-like list and dictionary combined.\n",
    "revs[0]  # Returns a dictionary containing information about the first revision of the page.\n",
    "         # The dictionary contains keys such as 'user', 'comment', 'timestamp', etc. that provide details about the revision.\n",
    "         # The specific information returned depends on the MediaWiki API and the parameters used to query the revisions.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7802a3c8-d1e2-40b9-a518-9f0bb5b375c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 29 µs, sys: 1e+03 ns, total: 30 µs\n",
      "Wall time: 32.7 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#\n",
    "#  BLOCK THREE\n",
    "#\n",
    "\n",
    "# Sort the list of revisions for the specified page in ascending order based on their timestamps.\n",
    "# The sorted() function is used here with a key parameter, which takes a function that returns a value to sort by.\n",
    "# In this case, the lambda function returns the timestamp value from each revision dictionary.\n",
    "revs = sorted(revs, key=lambda rev: rev['timestamp'])  # Returns the same list of revision dictionaries, but sorted\n",
    "                                                       # in ascending order by their timestamps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "95dbabb2-2b85-4d9b-b46f-c0501aed74d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('revid', 386591531),\n",
       "             ('parentid', 0),\n",
       "             ('user', 'Nirvana2013'),\n",
       "             ('timestamp',\n",
       "              time.struct_time(tm_year=2010, tm_mon=9, tm_mday=23, tm_hour=18, tm_min=49, tm_sec=7, tm_wday=3, tm_yday=266, tm_isdst=-1)),\n",
       "             ('comment', 'Add link')])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Look at the first revision in the sorted/reordered list of revisions for the specified page.\n",
    "# Outputs an ordered dictionary, which is a Python class that functions as a hybrid-like list and dictionary combined.\n",
    "revs[0]  \n",
    "# Returns a dictionary containing information about the first revision of the page in the sorted list."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69df3e90-ba03-4e12-922a-efd42d495c23",
   "metadata": {},
   "source": [
    "# Identifying the 'sentiment'of the wikipedia page edits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "22a64791-9264-4b74-90b6-2752a8d1111b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "All model checkpoint layers were used when initializing TFDistilBertForSequenceClassification.\n",
      "\n",
      "All the layers of TFDistilBertForSequenceClassification were initialized from the model checkpoint at distilbert-base-uncased-finetuned-sst-2-english.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertForSequenceClassification for predictions without further training.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 686 ms, sys: 270 ms, total: 956 ms\n",
      "Wall time: 1.92 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#\n",
    "# BLOCK FOUR\n",
    "#\n",
    "\n",
    "# This library provides for pipelines to 'deep learning' models; requires TensorFlow 2.0 or PyTorch\n",
    "# AutoModelForSequenceClassification provides a pre-trained sequence classification model\n",
    "# AutoTokenizer is used to automatically select the appropriate tokenizer for a given pre-trained model. \n",
    "# The tokenizer is responsible for preprocessing raw text data into a format understood by the deep learning model.\n",
    "# The pipeline function is used for running pre-trained models.\n",
    "from transformers import TFAutoModelForSequenceClassification, AutoTokenizer, pipeline\n",
    "\n",
    "\n",
    "# Load the pre-trained model and tokenizer\n",
    "model = TFAutoModelForSequenceClassification.from_pretrained(\"distilbert-base-uncased-finetuned-sst-2-english\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
    "\n",
    "# Create a sentiment analysis pipeline using the loaded model and tokenizer\n",
    "sentiment_pipeline = pipeline('sentiment-analysis',\n",
    "                              model=model,            # use the loaded model\n",
    "                              tokenizer=tokenizer,    # use the tokenizer\n",
    "                              max_length=256,         # set the maximum input length to a smaller value (e.g., 256)\n",
    "                              )\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0c724071-6fe2-487d-ba54-72263531ea64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1 µs, sys: 1e+03 ns, total: 2 µs\n",
      "Wall time: 1.91 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#\n",
    "# BLOCK FIVE\n",
    "#\n",
    "\n",
    "#\n",
    "def find_sentiment(text: str) -> float:\n",
    "    \"\"\"\n",
    "    This function takes in a string of text and returns a sentiment score between -1 and 1,\n",
    "    where negative values indicate negative sentiment and positive values indicate positive sentiment.\n",
    "    \"\"\"\n",
    "    # run the text through the sentiment analysis model and get the first (and only) result\n",
    "    sent: List[dict] = sentiment_pipeline([text[:250]])[0]\n",
    "    \n",
    "    # extract the sentiment score from the model output\n",
    "    score: float = sent['score']\n",
    "    \n",
    "    # check if the sentiment label is negative\n",
    "    if sent['label'] == 'NEGATIVE':\n",
    "        \n",
    "        # if so, multiply the score by -1 to indicate negative sentiment\n",
    "        score *= -1\n",
    "\n",
    "    # return the sentiment score    \n",
    "    return score\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5619cce7-5e87-4ecd-82d3-82b892c9da8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Provides for pipeline to 'deep learning' models; requires TensorFlow 2.0 or PyTorch\n",
    "# from transformers import pipeline  # import the transformers library, which includes the pipeline function for running pre-trained models\n",
    "\n",
    "# from typing import Dict, List, Tuple, Any  # import List and Tuple types from the typing module for type hinting\n",
    "\n",
    "# # sentiment_pipeline = pipeline(\"sentiment-analysis\")  # initialize the Deep Learning sentiment analysis model using the pipeline function from transformers\n",
    "\n",
    "# # initialize the Deep Learning sentiment analysis model using the pipeline function from transformers\n",
    "# sentiment_pipeline = pipeline('sentiment-analysis',\n",
    "#                               model='distilbert-base-uncased-finetuned-sst-2-english',\n",
    "#                               max_length=256)  # reduced maximum input length to a smaller value (e.g., 256)\n",
    "\n",
    "# def find_sentiment(text: str) -> float:\n",
    "#     \"\"\"\n",
    "#     This function takes in a string of text and returns a sentiment score between -1 and 1,\n",
    "#     where negative values indicate negative sentiment and positive values indicate positive sentiment.\n",
    "#     \"\"\"\n",
    "#     # run the text through the sentiment analysis model and get the first (and only) result\n",
    "#     sent: List[dict] = sentiment_pipeline([text[:250]])[0]\n",
    "    \n",
    "#     # extract the sentiment score from the model output\n",
    "#     score: float = sent['score']\n",
    "    \n",
    "#     # check if the sentiment label is negative\n",
    "#     if sent['label'] == 'NEGATIVE':\n",
    "        \n",
    "#         # if so, multiply the score by -1 to indicate negative sentiment\n",
    "#         score *= -1\n",
    "\n",
    "#     # return the sentiment score    \n",
    "#     return score\n",
    "\n",
    "# #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "87be060c-d3d6-408d-9894-24d79ce66a05",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9998656511306763\n",
      "-0.9928188920021057\n",
      "-0.9936189651489258\n",
      "CPU times: user 250 ms, sys: 259 ms, total: 509 ms\n",
      "Wall time: 326 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#\n",
    "# TESTING BLOCK\n",
    "#\n",
    "\n",
    "# Test sentiment function operation\n",
    "test_positive_sent = find_sentiment('I love you') \n",
    "print(test_positive_sent)\n",
    "\n",
    "test_negative_sent = find_sentiment('I despise you') \n",
    "print(test_negative_sent)\n",
    "\n",
    "test_neutral_sent = find_sentiment('I neutral you') \n",
    "print(test_neutral_sent)\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f3939afe-beec-413b-a19c-8796778506ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# TESTING BLOCK\n",
    "#\n",
    "\n",
    "# Test the sentiment analysis model\n",
    "# Define a list of text strings to analyze\n",
    "# texts = [\n",
    "#     \"This is a positive sentence.\",\n",
    "#     \"This is a negative sentence.\",\n",
    "#     \"This is a neutral sentence.\",\n",
    "# ]\n",
    "\n",
    "# Call the find_sentiment function to analyze the texts\n",
    "# sentiment_scores = find_sentiment(texts)\n",
    "\n",
    "# Print the sentiment scores\n",
    "# print(sentiment_scores)\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7a89017e-75d1-4f84-84b5-77aa110b244c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.93 s, sys: 8.7 s, total: 17.6 s\n",
      "Wall time: 11.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#\n",
    "#  BLOCK SIX\n",
    "#\n",
    "\n",
    "#\n",
    "from typing import Dict, List, Tuple, Any  # import List and Tuple types from the typing module for type hinting\n",
    "\n",
    "# Define the 'edits' dictionary\n",
    "edits: Dict[str, Dict[str, Any]] = {}\n",
    "\n",
    "# Iterate through the revisions in the sorted list 'revs'\n",
    "for rev in revs:\n",
    "    # Get the date of the revision using the 'timestamp' attribute\n",
    "    date: str = time.strftime('%Y-%m-%d', rev['timestamp'])\n",
    "    \n",
    "    # Check if the date is not in the 'edits' dictionary\n",
    "    if date not in edits:\n",
    "        # If the date is not in the 'edits' dictionary, create a new dictionary with the following attributes:\n",
    "        # - 'sentiments': a list to store the sentiment analysis scores\n",
    "        # - 'edt_count': an integer to store the number of edits made on the date\n",
    "        edits[date]: Dict[str, Any] = {'sentiments': [], 'edt_count': 0}\n",
    "        \n",
    "    # Increment the 'edt_count' by 1 for the current date\n",
    "    edits[date]['edt_count'] += 1\n",
    "        \n",
    "    # Get the comment for the revision, if it exists\n",
    "    comment: str = rev.get('comment', '')\n",
    "    \n",
    "    # Call the 'find_sentiment' function to get the sentiment score for the comment\n",
    "    sentiment_score: float = find_sentiment(comment)\n",
    "    \n",
    "    # Append the sentiment score to the 'sentiments' list for the current date\n",
    "    edits[date]['sentiments'].append(sentiment_score)\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8b0e65ae-8c69-40d1-a219-b2a82aa9f017",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.91 ms, sys: 1.69 ms, total: 4.61 ms\n",
      "Wall time: 4.68 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#\n",
    "#  BLOCK SEVEN\n",
    "#\n",
    "\n",
    "#\n",
    "from statistics import mean  # import mean function from statistics module\n",
    "\n",
    "# iterate through each key in the edits dictionary\n",
    "for key in edits:\n",
    "    \n",
    "    # check if there are any sentiments present for the current key\n",
    "    if len(edits[key]['sentiments']) > 0:\n",
    "        \n",
    "        # calculate the mean sentiment for the current key\n",
    "        edits[key]['sentiment'] = mean(edits[key]['sentiments'])\n",
    "        \n",
    "        # calculate the negative sentiment score as a ratio of the total sentiment scores\n",
    "        edits[key]['percent_neg_sentiment'] = len([s for s in edits[key]['sentiments'] if s < 0]) / len(edits[key]['sentiments'])\n",
    "                                           \n",
    "    # if there are no sentiments present for the current key\n",
    "    else:\n",
    "        \n",
    "        # set the sentiment score to 0\n",
    "        edits[key]['sentiment'] = 0\n",
    "                                           \n",
    "        # set the negative sentiment score to 0\n",
    "        edits[key]['neg_sentiment'] = 0\n",
    "\n",
    "        # remove the 'sentiments' key from the current key in the 'edits' dictionary\n",
    "        del edits[key]['sentiments']\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8d4aec21-51d8-4308-ba37-5b2aec4ff9b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'edits' is Key value Pair/s dictionary, each key => date\n",
    "# there are 3 key values; 'edt_count', 'sentiment', and 'neg_sentiment'\n",
    "# 'edt_count' = number of times BTC wikipedia page comments was edited on a given day\n",
    "# 'sentiment' = average sentiment for that day\n",
    "# 'percent_neg_sentiment' = percentage of edits that express sentiment negativity on that given day\n",
    "\n",
    "# edits"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f41da4b-2f6b-4fdc-821e-eb3820223136",
   "metadata": {},
   "source": [
    "# generate a dataframe with sentiment data\n",
    "transform -\n",
    "structure -\n",
    "clean-up dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8a82cf31-edb9-4775-9dcf-c9ae609409fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK EIGHT\n",
    "#\n",
    "\n",
    "# Import pandas module for creating and manipulating dataframes\n",
    "import pandas as pd\n",
    "\n",
    "# Create a dataframe 'edits_df' from the dictionary 'edits' with index orientation set to 'index'\n",
    "# takes in list of dictionaries, orient on 'index' ensures that each dictionary is a seperate row of the dataframe\n",
    "BTC_edits_df = pd.DataFrame.from_dict(edits, orient='index')  \n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d3cba34c-c31a-43f9-b6d0-9e4827e68437",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiments</th>\n",
       "      <th>edt_count</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>percent_neg_sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2010-09-23</th>\n",
       "      <td>[0.99639493227005]</td>\n",
       "      <td>1</td>\n",
       "      <td>0.996395</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2010-09-29</th>\n",
       "      <td>[0.9990505576133728]</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999051</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2010-09-30</th>\n",
       "      <td>[-0.9961720108985901]</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.996172</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2010-10-23</th>\n",
       "      <td>[-0.9994531273841858, -0.9997603297233582]</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.999607</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2010-10-26</th>\n",
       "      <td>[-0.9980478286743164]</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.998048</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentiments  edt_count  sentiment   \n",
       "2010-09-23                          [0.99639493227005]          1   0.996395  \\\n",
       "2010-09-29                        [0.9990505576133728]          1   0.999051   \n",
       "2010-09-30                       [-0.9961720108985901]          1  -0.996172   \n",
       "2010-10-23  [-0.9994531273841858, -0.9997603297233582]          2  -0.999607   \n",
       "2010-10-26                       [-0.9980478286743164]          1  -0.998048   \n",
       "\n",
       "            percent_neg_sentiment  \n",
       "2010-09-23                    0.0  \n",
       "2010-09-29                    0.0  \n",
       "2010-09-30                    1.0  \n",
       "2010-10-23                    1.0  \n",
       "2010-10-26                    1.0  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "BTC_edits_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9f7d254b-8083-4580-8deb-facf39c37481",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK NINE\n",
    "#\n",
    "\n",
    "# convert the index to a pandas 'datetime index' using pandas to_datetime() method\n",
    "BTC_edits_df.index = pd.to_datetime(BTC_edits_df.index)  # transforms from string into 'datetime object' \n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81308dcf-6354-4380-b582-da3b272956b8",
   "metadata": {},
   "source": [
    "# Addressing the problem of gaps in dates of activity in the dataframe\n",
    "there are data gaps within the dataframe\n",
    "\n",
    "i.e. asset or security is traded more often than days of comments or sentiment generating activity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e6ab02cb-896f-45d5-a299-1a522b47aeec",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatetimeIndex(['2009-03-08', '2009-03-09', '2009-03-10', '2009-03-11',\n",
       "               '2009-03-12', '2009-03-13', '2009-03-14', '2009-03-15',\n",
       "               '2009-03-16', '2009-03-17',\n",
       "               ...\n",
       "               '2023-05-09', '2023-05-10', '2023-05-11', '2023-05-12',\n",
       "               '2023-05-13', '2023-05-14', '2023-05-15', '2023-05-16',\n",
       "               '2023-05-17', '2023-05-18'],\n",
       "              dtype='datetime64[ns]', length=5185, freq='D')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\n",
    "# BLOCK TEN\n",
    "#\n",
    "\n",
    "from datetime import datetime\n",
    "# from datetime import datetime\n",
    "\n",
    "dates = pd.date_range(start= '2009-03-08', end= datetime.now())\n",
    "\n",
    "#\n",
    "#df = pd.DataFrame({'data':value}, index=pd.DatetimeIndex(timestamp), dtype=float))\n",
    "dates\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "68f4cd1d-fca1-48cc-ab6f-d3e8b796761f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiments</th>\n",
       "      <th>edt_count</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>percent_neg_sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2009-03-08</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2009-03-09</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2009-03-10</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2009-03-11</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2009-03-12</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           sentiments  edt_count  sentiment  percent_neg_sentiment\n",
       "2009-03-08          0          0        0.0                    0.0\n",
       "2009-03-09          0          0        0.0                    0.0\n",
       "2009-03-10          0          0        0.0                    0.0\n",
       "2009-03-11          0          0        0.0                    0.0\n",
       "2009-03-12          0          0        0.0                    0.0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\n",
    "# BLOCK ELEVEN\n",
    "#\n",
    "\n",
    "#\n",
    "BTC_edits_df = BTC_edits_df.reindex(dates, fill_value=0)\n",
    "BTC_edits_df.head()\n",
    "\n",
    "# BTC_edits_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ab9a99d8-9696-4e7a-b5f1-d0238d73d87a",
   "metadata": {},
   "outputs": [
    {
     "ename": "DataError",
     "evalue": "Cannot aggregate non-numeric type: object",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;31mTypeError\u001b[0m: float() argument must be a string or a real number, not 'list'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "File \u001b[0;32m/opt/miniconda3/envs/tensorflow/lib/python3.10/site-packages/pandas/core/window/rolling.py:368\u001b[0m, in \u001b[0;36mBaseWindow._prep_values\u001b[0;34m(self, values)\u001b[0m\n\u001b[1;32m    367\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 368\u001b[0m         values \u001b[38;5;241m=\u001b[39m \u001b[43mensure_float64\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    369\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mValueError\u001b[39;00m, \u001b[38;5;167;01mTypeError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[0;32mpandas/_libs/algos_common_helper.pxi:42\u001b[0m, in \u001b[0;36mpandas._libs.algos.ensure_float64\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: setting an array element with a sequence.",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "File \u001b[0;32m/opt/miniconda3/envs/tensorflow/lib/python3.10/site-packages/pandas/core/window/rolling.py:485\u001b[0m, in \u001b[0;36mBaseWindow._apply_blockwise\u001b[0;34m(self, homogeneous_func, name, numeric_only)\u001b[0m\n\u001b[1;32m    484\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 485\u001b[0m     arr \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_prep_values\u001b[49m\u001b[43m(\u001b[49m\u001b[43marr\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    486\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mTypeError\u001b[39;00m, \u001b[38;5;167;01mNotImplementedError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[0;32m/opt/miniconda3/envs/tensorflow/lib/python3.10/site-packages/pandas/core/window/rolling.py:370\u001b[0m, in \u001b[0;36mBaseWindow._prep_values\u001b[0;34m(self, values)\u001b[0m\n\u001b[1;32m    369\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mValueError\u001b[39;00m, \u001b[38;5;167;01mTypeError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 370\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcannot handle this type -> \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalues\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[1;32m    372\u001b[0m \u001b[38;5;66;03m# Convert inf to nan for C funcs\u001b[39;00m\n",
      "\u001b[0;31mTypeError\u001b[0m: cannot handle this type -> object",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mDataError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[24], line 5\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# convert non-numeric columns to numeric format\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;66;03m# BTC_edits_df['sentiments'] = pd.to_numeric(BTC_edits_df['sentiments'], errors='coerce')\u001b[39;00m\n\u001b[1;32m      3\u001b[0m \n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# create a rolling 30 day average of edits and sentiment\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m rolling_BTC_edits \u001b[38;5;241m=\u001b[39m \u001b[43mBTC_edits_df\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrolling\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m30\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmean\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/miniconda3/envs/tensorflow/lib/python3.10/site-packages/pandas/core/window/rolling.py:2111\u001b[0m, in \u001b[0;36mRolling.mean\u001b[0;34m(self, numeric_only, engine, engine_kwargs)\u001b[0m\n\u001b[1;32m   2068\u001b[0m \u001b[38;5;129m@doc\u001b[39m(\n\u001b[1;32m   2069\u001b[0m     template_header,\n\u001b[1;32m   2070\u001b[0m     create_section_header(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mParameters\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   2109\u001b[0m     engine_kwargs: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mbool\u001b[39m] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m   2110\u001b[0m ):\n\u001b[0;32m-> 2111\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmean\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2112\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnumeric_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnumeric_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2113\u001b[0m \u001b[43m        \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2114\u001b[0m \u001b[43m        \u001b[49m\u001b[43mengine_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mengine_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2115\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/miniconda3/envs/tensorflow/lib/python3.10/site-packages/pandas/core/window/rolling.py:1507\u001b[0m, in \u001b[0;36mRollingAndExpandingMixin.mean\u001b[0;34m(self, numeric_only, engine, engine_kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_numba_apply(sliding_mean, engine_kwargs)\n\u001b[1;32m   1506\u001b[0m window_func \u001b[38;5;241m=\u001b[39m window_aggregations\u001b[38;5;241m.\u001b[39mroll_mean\n\u001b[0;32m-> 1507\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwindow_func\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmean\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnumeric_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnumeric_only\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/miniconda3/envs/tensorflow/lib/python3.10/site-packages/pandas/core/window/rolling.py:615\u001b[0m, in \u001b[0;36mBaseWindow._apply\u001b[0;34m(self, func, name, numeric_only, numba_args, **kwargs)\u001b[0m\n\u001b[1;32m    612\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[1;32m    614\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmethod \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msingle\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 615\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_apply_blockwise\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhomogeneous_func\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnumeric_only\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    616\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    617\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_apply_tablewise(homogeneous_func, name, numeric_only)\n",
      "File \u001b[0;32m/opt/miniconda3/envs/tensorflow/lib/python3.10/site-packages/pandas/core/window/rolling.py:487\u001b[0m, in \u001b[0;36mBaseWindow._apply_blockwise\u001b[0;34m(self, homogeneous_func, name, numeric_only)\u001b[0m\n\u001b[1;32m    485\u001b[0m     arr \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prep_values(arr)\n\u001b[1;32m    486\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mTypeError\u001b[39;00m, \u001b[38;5;167;01mNotImplementedError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 487\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m DataError(\n\u001b[1;32m    488\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot aggregate non-numeric type: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00marr\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    489\u001b[0m     ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[1;32m    490\u001b[0m res \u001b[38;5;241m=\u001b[39m homogeneous_func(arr)\n\u001b[1;32m    491\u001b[0m res_values\u001b[38;5;241m.\u001b[39mappend(res)\n",
      "\u001b[0;31mDataError\u001b[0m: Cannot aggregate non-numeric type: object"
     ]
    }
   ],
   "source": [
    "# convert non-numeric columns to numeric format\n",
    "# BTC_edits_df['sentiments'] = pd.to_numeric(BTC_edits_df['sentiments'], errors='coerce')\n",
    "\n",
    "# create a rolling 30 day average of edits and sentiment\n",
    "rolling_BTC_edits = BTC_edits_df.rolling(30).mean()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad3e1d8d-ccd4-468c-84a6-54c0030d7947",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BTC_edits_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09f2c795-aacc-4aed-9bff-ec400f05bdf5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# convert column to numeric type, with any non-numeric values set to NaN\n",
    "BTC_edits_df[\"column_name\"] = pd.to_numeric(BTC_edits_df[\"column_name\"], errors='coerce')\n",
    "\n",
    "# create a rolling 30 day average of edits\n",
    "rolling_BTC_edits = BTC_edits_df.rolling(30).mean()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a997ed13-2fd7-406d-8160-330161f89338",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BTC_edits_df['sentiments'] = BTC_edits_df['sentiments'].apply(lambda x: [x]).apply(lambda x: x[0]).astype(float)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc1767c6-bf4b-4133-97dc-5a4bf9266541",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(BTC_edits_df['sentiments'].dtype)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5312589-6278-493a-b7cd-cef741457625",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BTC_edits_df['sentiments'] = BTC_edits_df['sentiments'].apply(lambda x: x[0]).astype(float)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "736e469f-e493-4474-bcef-6eee0ff168bb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BTC_edits_df['sentiments'] = BTC_edits_df['sentiments'].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c510ac72-62ab-42ec-a8a9-c2181299f44c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK TWELVE\n",
    "#\n",
    "\n",
    "# create a rolling 30 day average of edits and sentiment\n",
    "rolling_BTC_edits = BTC_edits_df.rolling(30).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9176c7ba-3197-4ae7-9111-6adea709e64e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d62d4ef-3f48-4789-9133-b2007ec4b067",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95bffb34-06a6-4729-ba53-c179126f26dd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcfa9ae8-83d6-4efb-b937-802757988e7f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80b731f5-adf0-4214-b4fd-be571e491919",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Exploratory data anlysis\n",
    "# Downloading assets/securities price data (i.e. S&P 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff841917-3d6b-4b4f-b967-d5650108944c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import yfinance package as package for data source\n",
    "import yfinance as yf  # call yahoo finance API in order to download daily stock and indices prices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0223d321-1d60-4b25-ad2f-0e2cf6e9641c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize a ticker class to enable download of pricing history for. single, i.e. specified symbol\n",
    "sp500 = yf.Ticker('^GSPC')  # GSPC is the S&P500 index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3606b87-471e-4156-8f82-7842fc6fc7f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to query historical prices\n",
    "sp500 = sp500.history(period = 'max') # 'max' allows to query ALL data from beginning of index creation in dataframe format\n",
    "\n",
    "# visualize pandas dataframe\n",
    "# each row representing data of a trading day with non-trading days excluded\n",
    "sp500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb443a84-c341-4a5d-9d1d-9c4801053993",
   "metadata": {},
   "outputs": [],
   "source": [
    "# examine the dataframe index; note returns a 'date-time-index'\n",
    "sp500.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbb40873-330e-4f50-bdd9-e32535e05f1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initiate preliminary data cleaning and visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6dd1645c-aa8e-4512-8975-f2702af9324c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# visualize data frame data via plotting closing price column against the index\n",
    "sp500.plot.line(y = 'Close', use_index = True)  # index values are x-axis and closing price is y-axis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "142ab74d-4512-4fd3-9d73-d435f3bedd96",
   "metadata": {},
   "outputs": [],
   "source": [
    "# eliminate columns that provide zero value for our analysis of the S&P500 index\n",
    "del sp500['Dividends']\n",
    "del sp500['Stock Splits']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1d318a4-d8c1-4df6-a15c-5932d1371ca9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define target for Machine Learning, i.e. what we will actually utilize Machine Leanring to predict\n",
    "# i.e. target / question is ~ will price go up or go down TOMORROW\n",
    "# note, predicting direction will be both more successful and more advantagious / greater utility than attempting to predict absolute price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "307b0c93-efe7-4025-be17-8cc2dce76600",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new column in our dataframe to provide tomorrow's price\n",
    "# creating new dataframe column called 'Tomorrow' \n",
    "# use pandas function '.shift()' to move the values of the 'Close' column one row ahead and \n",
    "# assign them to the 'Tomorrow' column, creating new column with the previous day's closing price\n",
    "sp500['Tomorrow'] = sp500['Close'].shift(-1)  \n",
    "\n",
    "\n",
    "# Visualize updated 'sp500' dataframe with new 'Tomorrow' column containing previous day's closing price.\n",
    "# allowing visualization if the market has gone up or down.\n",
    "sp500 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b7b2505-bda2-4e38-bf38-85eb222955ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Utilizing 'Tomorrow's' price, we will define our 'Target' as a new column in the dataframe\n",
    "# creating a new column named 'Target' in the dataframe.\n",
    "# comparing the values of the 'Tomorrow' column to the values of the 'Close' column\n",
    "# This generates a boolean (True or False) value depending on whether the 'Tomorrow' price is greater than the 'Close' price. \n",
    "sp500['Target'] = (sp500['Tomorrow'] > sp500['Close'])  \n",
    "\n",
    "# using the pandas '.astype()' function to convert the 'Target' column boolean to integers. \n",
    "sp500['Target'] = (sp500['Tomorrow'] > sp500['Close']).astype(int)  \n",
    "\n",
    "# Visualize updated 'sp500' dataframe with the new 'Target' column values changed from True or False to integer 1 or 0\n",
    "# value of 1 means 'Tomorrow' price was greater than the 'Close' price\n",
    "# value of 0 means that the 'Tomorrow' price was less than or equal to the 'Close' price.\n",
    "sp500 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfbb87ed-9734-4c44-b070-d3e12f92367c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean and/or streamline data by specifying a smaller range, i.e. how far back we will go for our starting point\n",
    "# Use pandas '.loc' method to specify using only those rows that are at least the specified date or later,\n",
    "# Use pandas '.copy()' method method returns a copy of the DataFrame, meaning (by default) any changes made in the original DataFrame will NOT be reflected in the copy.\n",
    "\n",
    "sp500 = sp500.loc['1990-01-01':].copy() \n",
    "# Here, we are using the pandas '.loc' method to specify that we only want to keep rows in the 'sp500' dataframe \n",
    "# that have a date of January 1st, 1990 or later. \n",
    "# We are then using the '.copy()' method to make a copy of the resulting dataframe so that any changes made to the \n",
    "# copy will not affect the original dataframe. This ensures that we are working with a clean and streamlined dataset.\n",
    "\n",
    "# Visualize the dataframe with the defined starting row \n",
    "sp500 \n",
    "# Here, we are displaying the updated 'sp500' dataframe with the defined starting row of January 1st, 1990. \n",
    "# This allows us to focus on more recent data and remove any older data that may not be relevant to our analysis.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "597610ff-f16f-489b-aea5-f23fe406915a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# develop initial Machine Learning Model, i.e. training a 'random forest' model\n",
    "# random forest, by design tendency towards resistance to 'overfitting'\n",
    "# random forest generally relatively quick in operation\n",
    "# random forest can identify or recognize 'non-linear' relationships of the data which is good in modeling securities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a806b444-5965-4dac-9253-a1964bed3e35",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import our first model package\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fcd6672-4027-44df-b01b-d15a5f3f545e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initializing the model\n",
    "# Set the random state, 'seed' to a specific value to ensure the same results are reproducible in the future\n",
    "seed = 42 \n",
    "\n",
    "\n",
    "# Create a Random Forest Classifier model with specified hyperparameters:\n",
    "# n_estimators specify the number of random decision trees will use; generally accuracy improves with higher number of decision trees\n",
    "#    setting the number of decision trees to 185, which is generally a good number for accuracy\n",
    "# min_samples_split helps prevent overfitting by specifying the minimum number of samples required to split an internal node; the higher this value the LOWER the accuracy,\n",
    "#    setting minimum number of samples required to split an internal node to 100, which helps to prevent overfitting.\n",
    "# random_state is used to ensure the same random values are used for each run of the model for reproducibility\n",
    "model = RandomForestClassifier(n_estimators=185, min_samples_split = 100, random_state = seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "996cec9e-5333-447b-9b6a-09a581c0314a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# split data into training and testing datasets\n",
    "# need to split taking 'time series' nature into account, as random splitting will result in leakage, i.e. using tomorrow's data to predict today\n",
    "train = sp500.iloc[:-100]  # training data set - use all data rows EXCEPT for the most recent/ last 100 rows\n",
    "test =  sp500.iloc[-100:]  # testing data set - use the most recent/ last 100 rows\n",
    "\n",
    "# select the columns that will be used to train the model\n",
    "predictors = ['Open', 'High', 'Low', 'Close', 'Volume']  # exclude the 'Target' and 'Tomorrow' columns to prevent leakage!\n",
    "\n",
    "# train the model using the specified predictors and target variables\n",
    "model.fit(train[predictors], train['Target'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b887fe9e-31e4-41d0-b182-f33a4a9fd173",
   "metadata": {},
   "outputs": [],
   "source": [
    "# use trained model using the '.predict()' method passing in 'test' set with the predictors to make predictions on the testing dataset\n",
    "preds = model.predict(test[predictors])\n",
    "\n",
    "# visualize predictions (note, default will be in numpy array format) \n",
    "preds\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fd95064-1571-4500-8fac-7ebab4f172b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# create new pandas series with predicted values using the 'preds' array and the index from the test dataset\n",
    "preds = pd.Series(preds, index = test.index)\n",
    "\n",
    "# visualize the predictions as a pandas series\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2d3b599-0cf9-438e-8dd4-0077526dea50",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assess the accuracy of the model - require metrics package/s\n",
    "from sklearn.metrics import (classification_report, \n",
    "                             balanced_accuracy_score,\n",
    "                             precision_score,\n",
    "                             confusion_matrix,\n",
    "                             roc_auc_score,\n",
    "                             f1_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f2c73f4-dac7-42a3-b007-a91faf2b8f18",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating the precision score using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "precision_score_1 = precision_score(test['Target'], preds)\n",
    "\n",
    "# visualize precision score\n",
    "print(precision_score_1)\n",
    "\n",
    "# Calculating the balanced accuracy score using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "balanced_accuracy_score_1 = balanced_accuracy_score(test['Target'], preds)\n",
    "\n",
    "# visualize balanced accuracy score\n",
    "print(balanced_accuracy_score_1)\n",
    "\n",
    "# Calculating the confusion matrix using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "confusion_matrix_1 = confusion_matrix(test['Target'], preds)\n",
    "\n",
    "# visualize confusion matrix\n",
    "print(confusion_matrix_1)\n",
    "\n",
    "# Calculating the classification report using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "classification_report_1 = classification_report(test['Target'], preds)\n",
    "\n",
    "# Printing the classification report\n",
    "print(classification_report_1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e8d1c8a-bd3a-41df-be51-02377f53b520",
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatenate the actual target values and predicted values into a new dataframe using concatenation; i.e. pd.concat()\n",
    "combined = pd.concat([test['Target'], preds], axis=1)  \n",
    "\n",
    "# visualize the combined values dataframe\n",
    "print(combined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27e58592-7eff-4c6a-9aad-4d33c290a4ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize the 'combined' dataframe using a line plot\n",
    "combined.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce96b386-82f1-4cdb-8f30-7ad34a9537b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# backtesting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14413bbe-edb2-474d-abaa-bc5952dc6468",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Any, List\n",
    "\n",
    "# functionalize the prediction model\n",
    "def predict(train: pd.DataFrame, test: pd.DataFrame, predictors: List[str], model: Any) -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Generates predictions for a test set using a trained machine learning model.\n",
    "    \n",
    "    Args:\n",
    "        train: DataFrame of the training data, including the target variable.\n",
    "        test: DataFrame of the test data.\n",
    "        predictors: List of feature names to use for making predictions.\n",
    "        model: A trained machine learning model object with a '.fit()' and '.predict()' method.\n",
    "    \n",
    "    Returns:\n",
    "        A DataFrame containing the actual target values and predicted values for the test set.\n",
    "    \"\"\"\n",
    "    \n",
    "    # fit/'train' model using the specified training data and predictor variables\n",
    "    model.fit(train[predictors], train['Target'])\n",
    "    \n",
    "    # generate predictions using the fitted model with the '.predict()' method passing in 'test' set with the predictors\n",
    "    preds = model.predict(test[predictors])\n",
    "    \n",
    "    # convert the numpy array of predicted values to a pandas Series with the same index as the test data\n",
    "    # name the Series \"Predictions\"\n",
    "    preds = pd.Series(preds, index = test.index, name = 'Predictions') \n",
    "    \n",
    "    # combine the actual target values and predicted values into a single DataFrame for comparison\n",
    "    combined = pd.concat([test['Target'], preds], axis = 1)\n",
    "    \n",
    "    # return the DataFrame with the actual and predicted values\n",
    "    return combined\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87b0f378-7daf-4067-ae63-7b890cf70b34",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function for backtesting the model\n",
    "# every trading year ~ 250 days, thus start of 2500 would be ~ 10 years of data; i.e. starts ~ 10 years back => use 10 years of data to train FIRST model\n",
    "# step is 250 which is ~ 1 year, ergo train the model for one year.  So, start by using 10 years data to train the 11th year, then 11 years of data to train the 12th year, etc...\n",
    "\n",
    "def backtest(data: pd.DataFrame, model: Any, predictors: List[str], start: int = 2500, step: int = 250) -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Backtests a machine learning model using rolling time windows.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    data: pd.DataFrame\n",
    "        DataFrame containing stock prices and other features.\n",
    "    model: Any\n",
    "        Machine learning model to use.\n",
    "    predictors: List[str]\n",
    "        List of features to use for making predictions.\n",
    "    start: int, optional\n",
    "        Index of the first row to start the backtest (default is 2500).\n",
    "    step: int, optional\n",
    "        Number of rows to include in each test set (default is 250).\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    pd.DataFrame\n",
    "        A concatenated DataFrame of all the predictions.\n",
    "    \"\"\"\n",
    "    \n",
    "    # Create an empty list to store all the predictions\n",
    "    all_predictions = []\n",
    "    \n",
    "    # Iterate over the specified range, starting from 'start' and stepping by 'step'; i.e. iterate through data year by year\n",
    "    for i in range(start, data.shape[0], step):\n",
    "        \n",
    "        # Split the data into training and test sets\n",
    "        train = data.iloc[0:i].copy() # The training set includes all rows up to the current index - i.e. all of years preceding current year\n",
    "        test =  data.iloc[i:(i+step)].copy() # The test set includes the next 'step' rows - i.e. current year\n",
    "        \n",
    "        # Use the predict function to make predictions for the test set\n",
    "        predictions = predict(train, test, predictors, model)\n",
    "        \n",
    "        # Append the predictions to the list of all predictions\n",
    "        all_predictions.append(predictions)\n",
    "    \n",
    "    # Concatenate all the predictions into a single DataFrame and return it\n",
    "    return pd.concat(all_predictions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6581a7b9-44c9-4200-acb2-2ca94cbe6cb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# backtesting the model with sp500 DataFrame, the model we trained earlier, and the list of predictors\n",
    "# the default start value of 2500 means that we start backtesting using data from 10 years ago\n",
    "# the default step value of 250 means that we backtest using one year of data at a time\n",
    "predictions = backtest(sp500, model, predictors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb417741-030f-4e8c-88ea-5ee915402ae0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# examine how many times predited market would go up and how many times predicted market would go down\n",
    "predictions['Predictions'].value_counts()  # '.value_counts()' will provide a numeric count for how many of each type of prediction was made"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6923899a-39a5-438c-8e92-d3289dea71ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# performance analytics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d564a655-4dd9-4ec6-a538-cf9f00227ca6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating the precision score using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "precision_score_2 = precision_score(predictions['Target'], predictions['Predictions'])\n",
    "\n",
    "# visualize precision score\n",
    "print(precision_score_2)\n",
    "\n",
    "# Calculating the balanced accuracy score using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "balanced_accuracy_score_2 = balanced_accuracy_score(predictions['Target'], predictions['Predictions'])\n",
    "\n",
    "# visualize balanced accuracy score\n",
    "print(balanced_accuracy_score_2)\n",
    "\n",
    "# Calculating the confusion matrix using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "confusion_matrix_2 = confusion_matrix(predictions['Target'], predictions['Predictions'])\n",
    "\n",
    "# visualize confusion matrix\n",
    "print(confusion_matrix_2)\n",
    "\n",
    "# Calculating the classification report using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "classification_report_2 = classification_report(predictions['Target'], predictions['Predictions'])\n",
    "\n",
    "# Printing the classification report\n",
    "print(classification_report_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6552155f-7742-4f34-8bf7-7273fdd5e845",
   "metadata": {},
   "outputs": [],
   "source": [
    "# percentage of days market actually went up\n",
    "predictions['Target'].value_counts() / predictions.shape[0] # value counts of the target divided by the total number of rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6be8453-4e49-4b97-9cd2-49b1ee8152fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# note above, actual performance of market was better than what we predicted; i.e. market rose ~53% and Machine Learning model predicted ~52%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd18b16a-458f-445c-9b06-60d1d1ea1052",
   "metadata": {},
   "outputs": [],
   "source": [
    "# expanding our predictors to use in model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8be84c25-3d30-4969-98c9-e75aadc06fe5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a list of horizons in days that will be used to calculate rolling averages of the S&P 500 stock index.\n",
    "# The list includes values of 2, 5, 60, 250, and 1000, which correspond to different periods of time: two days, one trading week (5 days), \n",
    "# three trading months (60 days), one trading year (250 days), and four trading years (1000 days).\n",
    "horizons = [2, 5, 60, 250, 1000]\n",
    "\n",
    "# Create an empty list to store the new predictor columns to be added to the dataset.\n",
    "new_predictors = []\n",
    "\n",
    "# Loop through each horizon value in the horizons list.\n",
    "for horizon in horizons:\n",
    "    # Calculate the rolling average of the S&P 500 stock index for the current horizon value.\n",
    "    rolling_averages = sp500.rolling(horizon).mean()\n",
    "    \n",
    "    # Create a new column name for the ratio of the closing price to the rolling average for the current horizon value.\n",
    "    ratio_column = f'Close_Ratio_{horizon}'\n",
    "    \n",
    "    # Calculate the ratio of the closing price to the rolling average for the current horizon value, \n",
    "    # and add it as a new column to the S&P 500 dataset.\n",
    "    sp500[ratio_column] = sp500['Close'] / rolling_averages['Close']\n",
    "    \n",
    "    # Create a new column name for the trend over the current horizon value.\n",
    "    trend_column = f'Trend_{horizon}'\n",
    "    \n",
    "    # Calculate the trend over the current horizon value by taking the sum of the 'Target' column over the previous horizon period \n",
    "    # and add it as a new column to the S&P 500 dataset.\n",
    "    # The shift(1) method shifts the data by 1 period so that the sum will be taken over the previous horizon period.\n",
    "    sp500[trend_column] = sp500.shift(1).rolling(horizon).sum()['Target']\n",
    "    \n",
    "    # Add the new ratio and trend column names to the new_predictors list.\n",
    "    new_predictors += [ratio_column, trend_column]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fd49675-a0e8-4ac0-8c67-91d17965c40f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop any rows in the `sp500` DataFrame that contain missing values (i.e., NaN values)\n",
    "sp500 = sp500.dropna()\n",
    "\n",
    "# Visualize the modified `sp500` DataFrame that has had missing value rows removed. \n",
    "# This allows us to inspect the remaining data and ensure that it is complete and appropriate for analysis.\n",
    "sp500\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "169f7afa-b8ad-4d47-9109-b51a96f96a74",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model improvement and optimizing - second model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3675dc22-0d8d-4d86-868e-5d8f44c15663",
   "metadata": {},
   "outputs": [],
   "source": [
    "# re-initialize our model (change model parameters)\n",
    "# set random state, i.e. seed to specific value to ensure repeatability given specified start point of randomness\n",
    "# n_estimators specify the number of random decision trees will use; generally accuracy improves with higher number of decision trees\n",
    "# min_samples_split aids in further preventing 'overfitting'; the higher this value the lower the accuracy, however the higher value will decrease risk of overfit\n",
    "\n",
    "seed = 42\n",
    "model = RandomForestClassifier(n_estimators=200, min_samples_split = 50, random_state = seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "891e8d89-a142-44f4-a927-6fa76403a83f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Update predict function; the prediction model\n",
    "\n",
    "def predict(train: pd.DataFrame, test: pd.DataFrame, predictors: List[str], model: Any) -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Train the specified model using the specified training data and predictor variables, and use the model to predict\n",
    "    the values of the 'Target' variable for the specified test data.\n",
    "    \n",
    "    Parameters:\n",
    "    train (pd.DataFrame): The training data used to fit the model.\n",
    "    test (pd.DataFrame): The test data used to generate predictions.\n",
    "    predictors (List[str]): A list of column names to use as predictor variables.\n",
    "    model (Any): The model to train and use for predictions.\n",
    "    \n",
    "    Returns:\n",
    "    pd.DataFrame: A DataFrame containing the actual and predicted values of the 'Target' variable.\n",
    "    \"\"\"\n",
    "    \n",
    "    # Train the model using the specified 'predictors' columns, then attempt to predict the 'Target'\n",
    "    model.fit(train[predictors], train['Target'])\n",
    "    \n",
    "    # Generate predictions with the model using the '.predict()' method passing in 'test' set with the predictors.\n",
    "    # Predictions are probabilities for the class with the highest probability.\n",
    "    # Extract the probabilities for the positive class, which is column 1 of the predictions array.\n",
    "    preds = model.predict_proba(test[predictors])[:,1]\n",
    "    \n",
    "    # Convert predicted probabilities to binary predictions:\n",
    "    # Set predictions >= 0.6 to 1, indicating an expected price increase\n",
    "    # Set predictions < 0.6 to 0, indicating an expected price decrease or no change\n",
    "    # these values will reduce number of actual trding days, thereby increase the chance or probability that day of trade price goes up\n",
    "    preds[preds >= 0.6] = 1\n",
    "    preds[preds < 0.6] = 0\n",
    "    \n",
    "    # Turn the numpy array into a pandas series and name it 'Predictions'\n",
    "    preds = pd.Series(preds, index = test.index, name = 'Predictions') \n",
    "    \n",
    "    # Generate a dataframe of actual values (i.e. test['Target']) and predicted values ('preds') concatenation\n",
    "    combined = pd.concat([test['Target'], preds], axis = 1)\n",
    "    \n",
    "    return combined\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1afca30a-bc09-4b58-80d0-6ae03ea74c8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# backtesting the S&P 500 data with the earlier created model and asociated predictors created earlier.\n",
    "predictions = backtest(sp500, model, new_predictors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a20241d4-f360-47a0-94d9-e7b6294a30d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions['Predictions'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97dcf932-faa9-417a-989f-5d4b3552b156",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating the precision score using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "precision_score_3 = precision_score(predictions['Target'], predictions['Predictions'])\n",
    "\n",
    "# visualize precision score\n",
    "print(precision_score_3)\n",
    "\n",
    "# Calculating the balanced accuracy score using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "balanced_accuracy_score_3 = balanced_accuracy_score(predictions['Target'], predictions['Predictions'])\n",
    "\n",
    "# visualize balanced accuracy score\n",
    "print(balanced_accuracy_score_3)\n",
    "\n",
    "# Calculating the confusion matrix using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "confusion_matrix_3 = confusion_matrix(predictions['Target'], predictions['Predictions'])\n",
    "\n",
    "# visualize confusion matrix\n",
    "print(confusion_matrix_3)\n",
    "\n",
    "# Calculating the classification report using predicted values 'preds' and actual values in the test set 'test['Target']'\n",
    "classification_report_3 = classification_report(predictions['Target'], predictions['Predictions'])\n",
    "\n",
    "# Printing the classification report\n",
    "print(classification_report_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "beb45c80-92f0-4d5b-a065-b30e08479529",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9 (tensorflow)",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}