-
Notifications
You must be signed in to change notification settings - Fork 0
/
getHistorical.py
81 lines (66 loc) · 2.97 KB
/
getHistorical.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
import logging
from binance.client import BinanceAPIException
from datetime import datetime
from pandas import pandas as pd
def millisToDateTimeString(millis):
from_obj = datetime.fromtimestamp(millis/1000.0)
return str(from_obj)
def getTicker(client, ticker_pair, interval, folder):
filename = folder+ticker_pair+interval+'.csv'
logging.info('Processing interval: '+interval+' for ticker :'+ticker_pair)
try:
df = pd.read_csv(filename)
file_found = True
stored_len = df.shape[0]
logging.debug('csv len: '+str(stored_len))
except FileNotFoundError as e:
file_found = False
logging.warn('File not found for pair: '+ticker_pair)
logging.warn(e)
try:
earliest_available = client._get_earliest_valid_timestamp(ticker_pair,
interval)
earliest_available_s = millisToDateTimeString(earliest_available)
logging.debug('earliest available api record: '+earliest_available_s)
except BinanceAPIException as e:
earliest_available = 0
logging.error('there was a BinanceAPIException '+str(e.message))
pass
if(file_found and earliest_available):
df.set_index('date', inplace=True)
index = df.index
max_date = index.max()
max_date_s = millisToDateTimeString(max_date)
logging.debug('latest register is: '+max_date_s)
if(earliest_available > max_date):
logging.warn('there will be a gap in the records between '
+max_date_s+' and '+earliest_available_s)
from_date_to_fetch = earliest_available
else:
from_date_to_fetch = max_date
else:
from_date_to_fetch = earliest_available
if from_date_to_fetch:
bars = client.get_historical_klines(ticker_pair, interval,
str(from_date_to_fetch),
limit=1000)
new_df = pd.DataFrame(bars, columns=['date', 'open', 'high', 'low',
'close', 'volume', 'close time',
'qav', 'num of trades',
'taker buy base asset vol',
'take buy quote asset vol',
'ignore'])
length = new_df.shape[0]
logging.info('Fetched data lenght: '+str(length))
new_df['hrd'] = [datetime.fromtimestamp(x / 1000.0) for x in new_df.date]
new_df.set_index('date', inplace=True)
if(file_found):
df = pd.concat([df, new_df])
else:
df = new_df
# export DataFrame to csv
logging.info('storing lines: '+str(df.shape[0])+
' for ticker '+ticker_pair)
df.to_csv(filename, index=True)
else:
logging.info('there was an error fetching data for ticker '+ticker_pair)