-
Notifications
You must be signed in to change notification settings - Fork 0
/
tasks.py
127 lines (109 loc) · 4.57 KB
/
tasks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
import logging
from celery.schedules import crontab
import datetime as dt
import time
from celery import Celery
from ftplib import FTP
celery = Celery('tasks')
from app.models import Stock
import os, sys
celery.conf.CELERY_BROKER = 'amqp://'
celery.conf.CELERY_BACKEND = 'amqp'
celery.conf.CELERY_TIMEZONE = 'UTC'
# THIS IS UTC TIME, which is +4 hours of EST
HOUR = 16
MINUTE = 10
# READ THIS...
#
# Start the broker.
# rabbitmq-server is located in /usr/local/sbin
# /usr/local/sbin/rabbitmq-server -detached
#
# Start Celery from the ~/Code/python/cf2 directory
# celery -app=tasks worker --loglevel=info -B
celery.conf.CELERYBEAT_SCHEDULE = {
'download_stock_files': {
'task': 'tasks.download_stock_files',
'schedule': crontab(hour=HOUR,minute=MINUTE)
},
'parse_stock_files': {
'task': 'tasks.parse_stock_files',
'schedule': crontab(hour=HOUR,minute=MINUTE+1)
}
#'calculate_indicators': {
# 'task': 'tasks.calculate_indicators',
# 'schedule': crontab(hour=HOUR,minute=MINUTE)
#}
}
@celery.task
def download_stock_files_task():
download_stock_files()
def download_stock_files():
logging.info('Begin downloading stock files...')
try:
ftp = FTP('ftp.nasdaqtrader.com')
ftp.login()
ftp.cwd('symboldirectory')
ftp.retrbinary('RETR nasdaqlisted.txt', open('app/static/symbols/nasdaqlisted.txt', 'wb').write)
ftp.retrbinary('RETR otherlisted.txt', open('app/static/symbols/otherlisted.txt', 'wb').write)
logging.info('Finished downloading stock files.')
except Exception as e:
logging.warning("Couldn't download the nightly NASDAQ/NYSE files. Error message: %s", e)
@celery.task
def parse_stock_files_task():
parse_stock_files()
def parse_stock_files():
'''File information here: http://www.nasdaqtrader.com/trader.aspx?id=symboldirdefs'''
logging.info('Begin parsing the files and refreshing stock data.')
parse_nasdaq('NASDAQ')
parse_other('NYSE')
logging.info('Finished parsing the files and refreshing stock data.')
def parse_nasdaq(market):
logging.info('Begin parsing %s file.', market)
path = os.path.join(os.path.dirname(__file__),'app/static/symbols/nasdaqlisted.txt')
with open(path, 'r') as inFile:
next(inFile) # ignore header
for line in inFile:
#time.sleep(1)
split= line.strip('\r\n').split('|')
# if it's not a test stock and not the last line
if split[3] != "Y" and "File Creation Time" not in split[0]:
symbol = split[0]
name = split[1]
logging.info('Fetching data. Market: %s, Symbol: %s, Company Name: %s', market, symbol, name)
create_or_update_stock(symbol, name, market)
logging.info('Finished parsing %s file.', market)
def create_or_update_stock(symbol, name, market):
stock = Stock.query.filter(Stock.symbol == symbol,
Stock.market=='NASDAQ').first()
if stock is None:
logging.info('New stock (not currently in our database): Market: %s, Symbol: %s, Company Name: %s', market, symbol, name)
stock = Stock(symbol=symbol,name=name,market="NASDAQ")
df = stock.get_dataframe()
if df is None or len(df) == 0:
logging.warning('Error retrieving Stock from the database (DataFrame is empty...): Stock.id: %s, Market: %s, Symbol: %s, Company Name: %s', stock.id, market, symbol, name)
else:
stock.calculate_indicators()
@celery.task
def calculate_indicators_task():
calculate_indicators()
def calculate_indicators():
logging.info('Begin Calculating indicators for all stocks.')
for stock in Stock.query.all():
logging.info('Updating indicators for %s [%s]', stock.symbol, stock.name)
stock.calculate_indicators()
def parse_other(market):
logging.info('Begin parsing %s file.', market)
path = os.path.join(os.path.dirname(__file__),'app/static/symbols/otherlisted.txt')
with open(path, 'r') as inFile:
next(inFile) # ignore header
for line in inFile:
#time.sleep(1)
split = line.strip('\r\n').split('|')
# if it's not a test stock and not the last line
if 'File Creation Time' not in split[0] and split[2] == 'N' and split[6] != 'Y':
symbol = split[7]
name = split[1]
logging.info('Fetching data. Market: %s, Symbol: %s, Company Name: %s', market, symbol, name)
create_or_update_stock(symbol, name, market)
logging.info('Finished parsing %s file.', market)