From d42e3a051b5ad10ccba69a8fe6f0430ae4f69dd3 Mon Sep 17 00:00:00 2001 From: Hirotaka Aoki <113173839+aoki-h-jp@users.noreply.github.com> Date: Sat, 2 Sep 2023 03:07:50 +0900 Subject: [PATCH] Visualize liquidation map mode="gross_value" --- .gitignore | 3 +- liqmap/draw.py | 231 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 208 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 2e331c2..eca53ce 100644 --- a/.gitignore +++ b/.gitignore @@ -159,4 +159,5 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -liqmap/*.csv \ No newline at end of file +liqmap/*.csv +liqmap/*.png diff --git a/liqmap/draw.py b/liqmap/draw.py index cc5e930..c1fd741 100644 --- a/liqmap/draw.py +++ b/liqmap/draw.py @@ -1,8 +1,12 @@ +# import standard libraries +import datetime + +# import third-party libraries import matplotlib.pyplot as plt import pandas as pd - -from liqmap.download import BinanceAggTradesDownload, BybitAggTradesDownloader -from liqmap.exceptions import ExchangeNotSupportedError +from rich.progress import track +from download import BinanceAggTradesDownload, BybitAggTradesDownloader +from exceptions import ExchangeNotSupportedError class HistoricalDraw: @@ -13,23 +17,35 @@ class HistoricalDraw: def __init__( self, symbol: str, - start_date: str, - end_date: str, + start_datetime: str, + end_datetime: str, exchange: str, - **kwargs, ) -> None: """ :param symbol: Trading symbol - :param start_date: Start date for drawing - :param end_date: End date for drawing + :param start_datetime: Start datetime for drawing + :param end_datetime: End datetime for drawing :param exchange: Exchange name - :param kwargs: Other parameters """ self._symbol = symbol - self._start_date = start_date - self._end_date = end_date + self._start_datetime = start_datetime + self._end_datetime = end_datetime self._exchange = exchange - self._kwargs = kwargs + self._downloaded_list = [] + + def _make_start_date(self) -> str: + """ + Make start date from string + :return: datetime + """ + return datetime.datetime.strptime(self._start_datetime, "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%d") + + def _make_end_date(self) -> str: + """ + Make end date from string + :return: datetime + """ + return datetime.datetime.strptime(self._end_datetime, "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%d") def _download(self) -> None: """ @@ -41,15 +57,15 @@ def _download(self) -> None: aggtrades = BinanceAggTradesDownload( destination_dir=".", download_symbol=self._symbol, - start_date=self._start_date, - end_date=self._end_date, + start_date=self._make_start_date(), + end_date=self._make_end_date(), ) elif self._exchange == "bybit": aggtrades = BybitAggTradesDownloader( destination_dir=".", download_symbol=self._symbol, - start_date=self._start_date, - end_date=self._end_date, + start_date=self._make_start_date(), + end_date=self._make_end_date(), ) else: raise ExchangeNotSupportedError( @@ -57,6 +73,29 @@ def _download(self) -> None: ) aggtrades.download_aggtrades() + def _make_prefix_list(self) -> list: + """ + Make prefix list + :return: list of prefix + """ + prefix_list = [] + if self._exchange == "binance": + for date in pd.date_range(self._make_start_date(), self._make_end_date()): + prefix_list.append( + f"data/futures/um/daily/aggTrades/{self._symbol}/{self._symbol}-aggTrades-{date.strftime('%Y-%m-%d')}.csv" + ) + elif self._exchange == "bybit": + for date in pd.date_range(self._make_start_date(), self._make_end_date()): + prefix_list.append( + f"bybit_data/trading/{self._symbol}/{self._symbol}{date.strftime('%Y-%m-%d')}.csv" + ) + else: + raise ExchangeNotSupportedError( + f"Exchange {self._exchange} is not supported." + ) + + return prefix_list + # format aggTrades def _format_aggtrade_dataframe(self, filepath: str) -> pd.DataFrame: """ @@ -67,32 +106,174 @@ def _format_aggtrade_dataframe(self, filepath: str) -> pd.DataFrame: # Merge aggTrades into one dataframe aggtrades = pd.read_csv(filepath) if self._exchange == "binance": - aggtrades["transact_time"] = pd.to_datetime( + headers = ["agg_trade_id", "price", "quantity", "first_trade_id", "last_trade_id", "transact_time", + "is_buyer_maker"] + if aggtrades.columns.tolist() != headers: + aggtrades = pd.read_csv(filepath, header=None) + aggtrades.columns = headers + aggtrades["timestamp"] = pd.to_datetime( aggtrades["transact_time"], unit="ms" ) aggtrades["price"] = aggtrades["price"].astype(float) - aggtrades["quantity"] = aggtrades["quantity"].astype(float) + aggtrades["size"] = aggtrades["quantity"].astype(float) aggtrades["is_buyer_maker"] = aggtrades["is_buyer_maker"].astype(bool) - df = aggtrades[["transact_time", "price", "quantity", "is_buyer_maker"]] + aggtrades["side"] = aggtrades["is_buyer_maker"].apply( + lambda x: "Buy" if x is False else "Sell" + ) + aggtrades["amount"] = aggtrades["price"] * aggtrades["size"] elif self._exchange == "bybit": - aggtrades["transact_time"] = pd.to_datetime( - aggtrades["transact_time"], unit="ms" + aggtrades["timestamp"] = pd.to_datetime( + aggtrades["timestamp"]*1000, unit="ms" ) aggtrades["price"] = aggtrades["price"].astype(float) - aggtrades["quantity"] = aggtrades["quantity"].astype(float) - aggtrades["is_buyer_maker"] = aggtrades["is_buyer_maker"].astype(bool) - df = aggtrades[["transact_time", "price", "quantity", "is_buyer_maker"]] + aggtrades["size"] = aggtrades["size"].astype(float) + aggtrades["side"] = aggtrades["side"].astype(str) + aggtrades["amount"] = aggtrades["price"] * aggtrades["size"] else: raise ExchangeNotSupportedError( f"Exchange {self._exchange} is not supported." ) + df = aggtrades[["timestamp", "price", "size", "side", "amount"]] + df = df.sort_values(by="timestamp") return df - def liquidation_map_from_historical(self, mode="", threshold=0): + def liquidation_map_from_historical(self, mode="gross_value", threshold_gross_value=100000): + """ + Draw liquidation map from historical data + :param mode: draw mode + :param threshold_gross_value: + :return: + """ # Downloading historical data self._download() # Formatting historical data + df_merged = pd.DataFrame() + for prefix in track(self._make_prefix_list(), description="Formatting data"): + df_prefix = self._format_aggtrade_dataframe(prefix) + df_merged = pd.concat([df_merged, df_prefix]) + + df_merged = df_merged.sort_values(by="timestamp") + df_merged = df_merged.reset_index(drop=True) + df_merged = df_merged[df_merged["timestamp"] <= self._end_datetime] + df_merged = df_merged[df_merged["timestamp"] >= self._start_datetime] # Visualize liquidation map + # mode: gross_value + df_buy = df_merged[df_merged["side"] == "Buy"] + df_sell = df_merged[df_merged["side"] == "Sell"] + + df_buy = df_buy[df_buy["amount"] >= threshold_gross_value] + df_sell = df_sell[df_sell["amount"] >= threshold_gross_value] + + df_buy['LossCut100x'] = df_buy['price'] * 0.99 + df_buy['LossCut50x'] = df_buy['price'] * 0.98 + df_buy['LossCut25x'] = df_buy['price'] * 0.96 + df_buy['LossCut10x'] = df_buy['price'] * 0.90 + + df_sell['LossCut100x'] = df_sell['price'] * 1.01 + df_sell['LossCut50x'] = df_sell['price'] * 1.02 + df_sell['LossCut25x'] = df_sell['price'] * 1.04 + df_sell['LossCut10x'] = df_sell['price'] * 1.10 + + fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, sharey=True, figsize=(9, 9)) + # draw price on ax1 + for i, dt in enumerate(df_buy.index): + ax1.scatter(dt, df_buy.iloc[i, 1], s=200, facecolor="None", edgecolors='g') + + ax1.plot(df_merged["timestamp"], df_merged['price'], c='k', label="price") + ax1.set_xlabel('datetime') + ax1.set_ylabel('price [USDT]') + ax1.tick_params(axis='x', labelrotation=45) + ax1.set_xlim([datetime.datetime.strptime(self._start_datetime, "%Y-%m-%d %H:%M:%S"), datetime.datetime.strptime(self._end_datetime, "%Y-%m-%d %H:%M:%S")]) + title = f'{self._symbol}\n{self._start_datetime} -> {self._end_datetime}' + if mode == "gross_value": + title += f"\nthreshold: >= {threshold_gross_value} [USDT]" + ax1.set_title(title) + + # Buy liquidation map on ax2 + df_losscut_10x = pd.DataFrame(columns=['price', 'amount']) + for i, dt in enumerate(df_buy.index): + df_losscut_10x.loc[len(df_losscut_10x)] = {'price': df_buy.iloc[i, 8], 'amount': df_buy.iloc[i, 4]} + + df_losscut_25x = pd.DataFrame(columns=['price', 'amount']) + for i, dt in enumerate(df_buy.index): + df_losscut_25x.loc[len(df_losscut_25x)] = {'price': df_buy.iloc[i, 7], 'amount': df_buy.iloc[i, 4]} + + df_losscut_50x = pd.DataFrame(columns=['price', 'amount']) + for i, dt in enumerate(df_buy.index): + df_losscut_50x.loc[len(df_losscut_50x)] = {'price': df_buy.iloc[i, 6], 'amount': df_buy.iloc[i, 4]} + + df_losscut_100x = pd.DataFrame(columns=['price', 'amount']) + for i, dt in enumerate(df_buy.index): + df_losscut_100x.loc[len(df_losscut_100x)] = {'price': df_buy.iloc[i, 5], 'amount': df_buy.iloc[i, 4]} + + current_price = df_merged.iloc[-1, 1] + + df_losscut_list = [df_losscut_10x, df_losscut_25x, df_losscut_50x, df_losscut_100x] + labels = ["10x Leveraged", "25x Leveraged", "50x Leveraged", "100x Leveraged"] + colors = ["r", "g", "b", "y"] + tick_degits = 0 + max_amount = 0 + for i, df_losscut in enumerate(df_losscut_list): + df_losscut = df_losscut[df_losscut['price'] <= current_price] + g_ids = int((round(df_losscut['price'].max(), tick_degits) - round(df_losscut['price'].min(), tick_degits))) + bins = [round(round(df_losscut['price'].min(), tick_degits) + i * 10 ** -tick_degits, tick_degits) for i in range(g_ids)] + df_losscut['group_id'] = pd.cut(df_losscut['price'], bins=bins) + agg_df = df_losscut.groupby('group_id').sum() + ax2.barh([f.left for f in agg_df.index], agg_df['amount'], height=10 ** -tick_degits, color=colors[i], label=labels[i]) + if agg_df['amount'].max() > max_amount: + max_amount = agg_df['amount'].max() + + # Save liquidation map data as csv + save_title = f"{self._symbol}_{self._start_datetime.replace(' ', '_').replace(':', '-')}-{self._end_datetime.replace(' ', '_').replace(':', '-')}_{mode}_{threshold_gross_value}.png" + for df_l, label in zip(df_losscut_list, labels): + df_l.to_csv(f"{save_title.replace('.png', '')}_{label.replace(' ','_')}_buy.csv") + + # Sell liquidation map on ax2 + df_losscut_10x = pd.DataFrame(columns=['price', 'amount']) + for i, dt in enumerate(df_sell.index): + df_losscut_10x.loc[len(df_losscut_10x)] = {'price': df_sell.iloc[i, 8], 'amount': df_sell.iloc[i, 4]} + + df_losscut_25x = pd.DataFrame(columns=['price', 'amount']) + for i, dt in enumerate(df_sell.index): + df_losscut_25x.loc[len(df_losscut_25x)] = {'price': df_sell.iloc[i, 7], 'amount': df_sell.iloc[i, 4]} + + df_losscut_50x = pd.DataFrame(columns=['price', 'amount']) + for i, dt in enumerate(df_sell.index): + df_losscut_50x.loc[len(df_losscut_50x)] = {'price': df_sell.iloc[i, 6], 'amount': df_sell.iloc[i, 4]} + + df_losscut_100x = pd.DataFrame(columns=['price', 'amount']) + for i, dt in enumerate(df_sell.index): + df_losscut_100x.loc[len(df_losscut_100x)] = {'price': df_sell.iloc[i, 5], 'amount': df_sell.iloc[i, 4]} + + current_price = df_merged.iloc[-1, 1] + + df_losscut_list = [df_losscut_10x, df_losscut_25x, df_losscut_50x, df_losscut_100x] + labels = ["10x Leveraged", "25x Leveraged", "50x Leveraged", "100x Leveraged"] + colors = ["r", "g", "b", "y"] + tick_degits = 0 + max_amount = 0 + for i, df_losscut in enumerate(df_losscut_list): + df_losscut = df_losscut[df_losscut['price'] >= current_price] + g_ids = int((round(df_losscut['price'].max(), tick_degits) - round(df_losscut['price'].min(), tick_degits))) + bins = [round(round(df_losscut['price'].min(), tick_degits) + i * 10 ** -tick_degits, tick_degits) for i in range(g_ids)] + df_losscut['group_id'] = pd.cut(df_losscut['price'], bins=bins) + agg_df = df_losscut.groupby('group_id').sum() + ax2.barh([f.left for f in agg_df.index], agg_df['amount'], height=10 ** -tick_degits, color=colors[i]) + if agg_df['amount'].max() > max_amount: + max_amount = agg_df['amount'].max() + + ax2.hlines(y=current_price, xmin=0, xmax=max_amount, linestyle='-.', colors='k') + ax2.set_title('Estimated Liquidation Amount') + ax2.set_xlabel('Amount') + ax2.tick_params(axis='x', labelrotation=45) + plt.legend(loc='upper right') + plt.tight_layout() + plt.savefig(save_title) + plt.close() + + # Save liquidation map data as csv + for df_l, label in zip(df_losscut_list, labels): + df_l.to_csv(f"{save_title.replace('.png', '')}_{label.replace(' ','_')}_sell.csv")