/
ftx.py
208 lines (167 loc) · 6.51 KB
/
ftx.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
'''
Copyright (C) 2017-2019 Bryant Moscon - bmoscon@gmail.com
Please see the LICENSE file for the terms and conditions
associated with this software.
'''
import requests
import logging
from time import sleep
import pandas as pd
from sortedcontainers.sorteddict import SortedDict as sd
from cryptofeed.rest.api import API, request_retry
from cryptofeed.defines import FTX as FTX_ID, SELL, BUY, BID, ASK
from cryptofeed.standards import pair_std_to_exchange
LOG = logging.getLogger('rest')
RATE_LIMIT_SLEEP = 0.2
class FTX(API):
ID = FTX_ID
api = "https://ftx.com/api"
def _get(self, command: str, params=None, retry=None, retry_wait=0):
url = f"{self.api}{command}"
@request_retry(self.ID, retry, retry_wait)
def helper():
resp = requests.get(url, params={} if not params else params)
self._handle_error(resp, LOG)
return resp.json()
return helper()
def ticker(self, symbol: str, retry=None, retry_wait=0):
sym = pair_std_to_exchange(symbol, self.ID)
data = self._get(f"/markets/{sym}", retry=retry, retry_wait=retry_wait)
return {'pair': symbol,
'feed': self.ID,
'bid': data['result']['bid'],
'ask': data['result']['ask']
}
def l2_book(self, symbol: str, retry=None, retry_wait=0):
sym = pair_std_to_exchange(symbol, self.ID)
data = self._get(f"/markets/{sym}/orderbook", {'depth': 100}, retry=retry, retry_wait=retry_wait)
return {
BID: sd({
u[0]: u[1]
for u in data['result']['bids']
}),
ASK: sd({
u[0]: u[1]
for u in data['result']['asks']
})
}
def trades(self, symbol: str, start=None, end=None, retry=None, retry_wait=10):
symbol = pair_std_to_exchange(symbol, self.ID)
for data in self._get_trades_hist(symbol, start, end, retry, retry_wait):
yield data
def funding(self, symbol: str, start_date=None, end_date=None, retry=None, retry_wait=10):
last = []
start = None
end = None
if end_date and not start_date:
start_date = '2019-01-01'
if start_date:
if not end_date:
end_date = pd.Timestamp.utcnow()
start = API._timestamp(start_date)
end = API._timestamp(end_date)
start = int(start.timestamp())
end = int(end.timestamp())
@request_retry(self.ID, retry, retry_wait)
def helper(start, end):
if start and end:
return requests.get(f"{self.api}/funding_rates?future={symbol}&start_time={start}&end_time={end}")
else:
return requests.get(f"{self.api}/funding_rates?symbol={symbol}")
while True:
r = helper(start, end)
if r.status_code == 429:
sleep(RATE_LIMIT_SLEEP)
continue
elif r.status_code == 500:
LOG.warning("%s: 500 for URL %s - %s", self.ID, r.url, r.text)
sleep(retry_wait)
continue
elif r.status_code != 200:
self._handle_error(r, LOG)
else:
sleep(RATE_LIMIT_SLEEP)
data = r.json()['result']
if data == []:
LOG.warning("%s: No data for range %d - %d", self.ID, start, end)
else:
end = int(API._timestamp(data[-1]["time"]).timestamp()) + 1
orig_data = list(data)
# data = self._dedupe(data, last)
# last = list(orig_data)
data = [self._funding_normalization(x, symbol) for x in data]
return data
@staticmethod
def _dedupe(data, last):
if len(last) == 0:
return data
ids = set([data['id'] for data in last])
ret = []
for d in data:
if d['id'] in ids:
continue
ids.add(d['id'])
ret.append(d)
return ret
def _get_trades_hist(self, symbol, start_date, end_date, retry, retry_wait):
last = []
start = None
end = None
if end_date and not start_date:
start_date = '2019-01-01'
if start_date:
if not end_date:
end_date = pd.Timestamp.utcnow()
start = API._timestamp(start_date)
end = API._timestamp(end_date)
start = int(start.timestamp())
end = int(end.timestamp())
@request_retry(self.ID, retry, retry_wait)
def helper(start, end):
if start and end:
return requests.get(f"{self.api}/markets/{symbol}/trades?limit=100&start_time={start}&end_time={end}")
else:
return requests.get(f"{self.api}/markets/{symbol}/trades")
while True:
r = helper(start, end)
if r.status_code == 429:
sleep(RATE_LIMIT_SLEEP)
continue
elif r.status_code == 500:
LOG.warning("%s: 500 for URL %s - %s", self.ID, r.url, r.text)
sleep(retry_wait)
continue
elif r.status_code != 200:
self._handle_error(r, LOG)
else:
sleep(RATE_LIMIT_SLEEP)
data = r.json()['result']
if data == []:
LOG.warning("%s: No data for range %d - %d", self.ID, start, end)
else:
end = int(API._timestamp(data[-1]["time"]).timestamp()) + 1
orig_data = list(data)
data = self._dedupe(data, last)
last = list(orig_data)
data = [self._trade_normalization(x, symbol) for x in data]
yield data
if len(orig_data) < 100:
break
def _trade_normalization(self, trade: dict, symbol: str) -> dict:
return {
'timestamp': API._timestamp(trade['time']).timestamp(),
'pair': symbol,
'id': trade['id'],
'feed': self.ID,
'side': SELL if trade['side'] == 'sell' else BUY,
'amount': trade['size'],
'price': trade['price']
}
def _funding_normalization(self, funding: dict, symbol: str) -> dict:
ts = pd.to_datetime(funding['time'], format="%Y-%m-%dT%H:%M:%S%z")
return {
'timestamp': API._timestamp(funding['time']).timestamp(),
'pair': funding['future'],
'feed': self.ID,
'rate': funding['rate']
}