Merge pull request #32 from devfinwiz/feature/caching-mechanism

[WIP] Enhanced Performance: Implementing Redis Caching for Sentiment Analysis Module

.dockerignore

@@ -1,4 +1,4 @@
-__pycache__
+**/__pycache__
 .pytest_cache
 .mypy_cache
 .git

Dockerfile

@@ -1,5 +1,15 @@
 FROM python:3.9-slim
 
+RUN apt-get update && apt-get install -y redis-server && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY redis.conf /etc/redis/redis.conf
+COPY entrypoint.sh /usr/local/bin/
+COPY shutdown.sh /usr/local/bin/
+
+VOLUME /redis-cache
+
 WORKDIR /app
 
 COPY pyproject.toml poetry.lock /app/
@@ -11,7 +21,15 @@ RUN pip install --no-cache-dir poetry && \
 COPY . .
 
 EXPOSE 8000
+EXPOSE 6379
+
+ENV REDIS_HOST=localhost
+ENV REDIS_PORT=6379
 
 WORKDIR /app/fin_maestro_kin
 
-CMD ["poetry", "run", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/shutdown.sh
+
+RUN touch /var/log/redis/redis-server.log && chown redis:redis /var/log/redis/redis-server.log
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+cd /app/fin_maestro_kin
+
+redis-server /etc/redis/redis.conf
+
+poetry run uvicorn main:app --host 0.0.0.0 --port 8000

fin_maestro_kin/modules/data_toolkit/nse/nse_operations.py

@@ -377,7 +377,7 @@ def index_pe_pb_div(self, symbol, start_date, end_date):
         end_date = datetime.datetime.strptime(end_date, "%d-%b-%Y").strftime("%d %b %Y")
 
         data = {"cinfo": f"{{'name':'{symbol}','startDate':'{start_date}','endDate':'{end_date}'}}"}
-        payload = requests.post('https://niftyindices.com/Backpage.aspx/getpepbHistoricaldataDBtoString', headers=self.niftyindices_headers,  json=data).json()
+        payload = requests.post('https://niftyindices.com/Backpage.aspx/getpepbHistoricaldataDBtoString', headers=self.niftyindices_headers, json=data).json()
         payload = json.loads(payload["d"])
 
         if not payload:
@@ -403,7 +403,7 @@ def index_total_returns(self, symbol,start_date,end_date):
         end_date = datetime.datetime.strptime(end_date, "%d-%b-%Y").strftime("%d %b %Y")
 
         data = {"cinfo": f"{{'name':'{symbol}','startDate':'{start_date}','endDate':'{end_date}'}}"}
-        payload = requests.post('https://niftyindices.com/Backpage.aspx/getTotalReturnIndexString', headers=self.niftyindices_headers,  json=data).json()
+        payload = requests.post('https://niftyindices.com/Backpage.aspx/getTotalReturnIndexString', headers=self.niftyindices_headers, json=data).json()
         payload = json.loads(payload["d"])
 
         if not payload:

fin_maestro_kin/modules/sentiment_analysis/sentiment_analysis.py

@@ -1,7 +1,8 @@
 import requests
 import json
+from datetime import datetime, time
 from fastapi import APIRouter, Query
-
+import redis
 
 class PCR():
     headers = {
@@ -10,8 +11,23 @@ class PCR():
         'accept-language': 'en-US,en;q=0.9'
     }
 
+    @staticmethod
+    def is_market_hours():
+        current_time = datetime.now().time()
+        return (current_time >= time(9, 0) and current_time <= time(15, 30))
+
+    @staticmethod
+    def get_cache_expiry():
+        if PCR.is_market_hours():
+            return 900
+        return 86400 
+
     @staticmethod
     def pcr_indice_scraper(symbol):
+        cached_result = redis_client.get(f"pcr_indice_{symbol}")
+        if cached_result:
+            return json.loads(cached_result)
+
         url = 'https://www.nseindia.com/api/option-chain-indices?symbol=' + symbol
         request = requests.get("https://www.nseindia.com", timeout=10, headers=PCR.headers)
         cookies = dict(request.cookies)
@@ -20,10 +36,18 @@ def pcr_indice_scraper(symbol):
         totCE = data['filtered']['CE']['totOI']
         totPE = data['filtered']['PE']['totOI']
         pcr = totPE / totCE
-        return round(pcr, 3)
+        pcr = round(pcr, 3)
+
+        redis_client.setex(f"pcr_indice_{symbol}", PCR.get_cache_expiry(), json.dumps(pcr))
+
+        return pcr
 
     @staticmethod
     def pcr_stocks_scraper(symbol):
+        cached_result = redis_client.get(f"pcr_stocks_{symbol}")
+        if cached_result:
+            return json.loads(cached_result)
+
         url = 'https://www.nseindia.com/api/option-chain-equities?symbol=' + symbol
         request = requests.get("https://www.nseindia.com", timeout=10, headers=PCR.headers)
         cookies = dict(request.cookies)
@@ -32,7 +56,11 @@ def pcr_stocks_scraper(symbol):
         totCE = data['filtered']['CE']['totOI']
         totPE = data['filtered']['PE']['totOI']
         pcr = totPE / totCE
-        return round(pcr, 3)
+        pcr = round(pcr, 3)
+
+        redis_client.setex(f"pcr_stocks_{symbol}", PCR.get_cache_expiry(), json.dumps(pcr))
+
+        return pcr
 
 
 class SentimentAnalyzer(PCR):
@@ -95,3 +123,5 @@ def get_state(pcr_value, thresholds):
             if pcr_value >= threshold:
                 return label
         return "Oversold"
+
+redis_client = redis.Redis(host='localhost', port=6379, db=0)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fin-maestro-kin"
-version = "0.2.6"
+version = "0.3.0"
 description = "Seamless Finance: Docker Deployed APIs for Smart Investments."
 authors = ["DEV_FINWIZ <devjuneja43@gmail.com>"]
 license = "MIT"
@@ -17,6 +17,7 @@ matplotlib = "^3.8.2"
 pandas = "^2.1.4"
 numpy = "^1.26.3"
 beautifulsoup4 = "^4.12.3"
+redis = "^5.0.4"
 
 [tool.poetry.dev-dependencies]
 pytest = "^8.1.1"