-
Notifications
You must be signed in to change notification settings - Fork 41
/
mc_environment_deep.py
713 lines (618 loc) · 25.1 KB
/
mc_environment_deep.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
from abc import ABC
import pkg_resources
# pkg_resources.require("gym==0.22.0")
import gym
import numpy as np
import time
from environments.mc_model.lob_utils.lob_functions import LOB
from environments.mc_model.mc_lob_simulation_class import MarkovChainLobModel
import matplotlib.pyplot as plt
import torch as th
import dill as pickle
import random
class MonteCarloEnvDeep(gym.Env, ABC):
"""
Parameters
----------
num_levels : int
number of price levels to consider in LOB. num_levels = number of price levels above best bid and best ask to
include
include_spread_levels : bool
argument passed on to LOB class, whether to include the spread levels in the LOB arrays
max_quote_depth : int
number of ticks away from best bid and best ask the MM can place its orders
T : int
time horizon
dt : float
time step size
mm_priority : bool
whether MM's orders should have highest priority or not. note that False does not result in a time priority
system, but instead the MM's orders have the last priority
phi : float
running inventory penalty parameter
reward_scale : float
a number with which all rewards are scaled
pre_run_on_start : bool
whether to perform a pre-run as an object is created / the environment is reset
pre_run_iterations : int
number of simulation events to include in the pre-run
debug : bool
whether or not information for debugging should be printed during simulation
randomize_reset : bool
if the LOB should get a random state after every reset
default_order_size : int
the size of the orders the MM places
Returns
-------
None
"""
def __init__(
self,
num_levels=10,
include_spread_levels=True,
max_quote_depth=5,
T=5000,
dt=1,
mm_priority=True,
phi=0,
reward_scale=1,
pre_run_on_start=False,
pre_run_iterations=int(1e4),
debug=False,
randomize_reset=True,
default_order_size=5,
):
super(MonteCarloEnvDeep, self).__init__()
self.include_spread_levels = include_spread_levels
self.num_levels = num_levels
self.starts_database = self.load_database()
self.mc_model = MarkovChainLobModel(
num_levels=self.num_levels,
ob_start=self.starts_database[0],
include_spread_levels=self.include_spread_levels,
)
self.pre_run_on_start = pre_run_on_start
self.pre_run_iterations = pre_run_iterations
if self.pre_run_on_start:
self.pre_run(self.pre_run_iterations)
self.mm_priority = mm_priority
self.buy_mo_sizes = list()
self.tot_vol_sell = list()
self.sell_mo_sizes = list()
self.tot_vol_buy = list()
# Environment-defining variables
self.max_quote_depth = max_quote_depth
self.phi = phi
self.T = T
self.dt = dt
self.num_action_times = int(self.T / self.dt)
self.t = 0
self.reward_scale = reward_scale
# Agent and environment variables
self.X_t = 0
self.X_t_previous = 0
self.Q_t = 0
self.H_t = 0
self.H_t_previous = 0
self.default_order_size = default_order_size
self.randomize_reset = randomize_reset
self.num_errors = 0
self.debug = debug
# Initiating MM LO variables
self.quotes_depths = {"bid": 1, "ask": 1}
self.quotes_absolute_level = {
"bid": self.mc_model.ob.ask - self.quotes_depths["bid"],
"ask": self.mc_model.ob.bid + self.quotes_depths["ask"],
}
self.order_volumes = {"bid": 0, "ask": 0}
self.set_action_space()
self.set_observation_space()
def load_database(
self, file_name="environments/mc_model/starts_database/start_bank_n100000.pkl"
):
"""
loads a database with random LOB states
Parameters
----------
file_name : str
where the database is stored
Returns
-------
lob_data_base : list
a list with random LOB states
"""
file = open(file_name, "rb")
lob_data_base = pickle.load(file)
return lob_data_base
def set_action_space(self):
"""
Sets the action space. The action space consists of the order depths and the action to place a market order.
The bid and ask depths are represented as a single integer.
Parameters
----------
None
Returns
-------
None
"""
self.action_space = gym.spaces.Discrete(self.max_quote_depth**2)
def _get_action_space_shape(self):
"""
Returns the shape of the action space as a tuple
"""
return self.action_space.shape
def set_observation_space(self):
"""
Creates the observation space
Parameters
----------
None
Returns
-------
None
"""
obs_dim = 3 + 2 * self.num_levels
low = -np.inf * np.ones(obs_dim)
high = np.inf * np.ones(obs_dim)
self.observation_space = gym.spaces.Box(low=low, high=high)
def state(self):
"""
Returns the current observable state
Parameters
----------
None
Returns
-------
obs : tuple
the observation space in terms of (time_varible, inventory_variable, spread, full LOB)
"""
lob = self.mc_model.ob.data[:, 1:].flatten() / 40
inv = self.Q_t / 100
t = self.t / self.T
spread = self.mc_model.ob.spread / 10
state = np.concatenate([np.array([inv, t, spread]), lob])
return state
def step(self, action: int):
"""
Receives an action (bid and ask depths) and takes a step in the environment accordingly
Parameters
----------
action : int
the bid and ask depths represented as a single integer
Returns
-------
obs : tuple
the observation space in terms of (time_varible, inventory_variable)
reward : float
the reward received at time t+1
done : bool
whether or not the episode is finished
"""
self.X_t_previous = self.X_t
self.H_t_previous = self.H_t
if self.t < self.T - self.dt:
# --------------- Placing limit orders in the order book ---------------
mm_bid_depth, mm_ask_depth = (
int(action / self.max_quote_depth) + 1,
action % self.max_quote_depth + 1,
)
self.order_volumes["bid"] = self.default_order_size
self.order_volumes["ask"] = self.default_order_size
# We need to specify the bid and ask prices before we start placing orders, otherwise we might alter them
ask = self.mc_model.ob.ask
bid = self.mc_model.ob.bid
# If invalid depths make adjustments
if self.mc_model.ob.spread >= mm_bid_depth + mm_ask_depth:
diff = self.mc_model.ob.spread - (mm_ask_depth + mm_bid_depth) + 1
mm_bid_depth += int(np.ceil(diff / 2))
mm_ask_depth += int(np.ceil(diff / 2))
# Placing the orders
self.quotes_absolute_level["bid"] = ask - mm_bid_depth
self.quotes_absolute_level["ask"] = bid + mm_ask_depth
self.mc_model.ob.change_volume(
level=ask - mm_bid_depth,
volume=-self.default_order_size,
absolute_level=True,
)
self.mc_model.ob.change_volume(
level=bid + mm_ask_depth,
volume=self.default_order_size,
absolute_level=True,
)
# The total volumes on market maker's bid and ask levels
# IMPORTANT: these quantities must be computed BEFORE simulation occurs
vol_on_mm_bid = -self.mc_model.ob.get_volume(
self.quotes_absolute_level["bid"], absolute_level=True
)
vol_on_mm_ask = self.mc_model.ob.get_volume(
self.quotes_absolute_level["ask"], absolute_level=True
)
else:
# --------------- Liquidating the inventory ---------------
if self.debug:
print("Liquidating the inventory")
print(self.mc_model.ob.data)
print(f"The bid volumes: {self.mc_model.ob.q_bid()}")
print(f"The ask volumes: {self.mc_model.ob.q_ask()}")
print(f"MM's inventory before liquidation: {self.Q_t}")
can_trade = True
if self.Q_t > 0:
level = self.mc_model.ob.bid
else:
level = self.mc_model.ob.ask
while can_trade:
# N.B. If the while-loop is traversed through more than once it means that the market maker has to
# walk the book in order to liquidate all of its holdings
if self.Q_t > 0:
volume_to_sell_current_trade = np.min(
[self.Q_t, self.mc_model.ob.bid_volume]
)
if (
self.mc_model.ob.bid_volume == 0
and self.mc_model.ob.outside_volume != 0
):
volume_to_sell_current_trade = self.mc_model.ob.outside_volume
elif (
self.mc_model.ob.bid_volume == 0
and self.mc_model.ob.outside_volume == 0
):
can_trade = False
if can_trade:
if np.sum(self.mc_model.ob.q_bid()) == 0:
trade_turnover = level * self.mc_model.ob.outside_volume
else:
trade_turnover = self.mc_model.ob.sell_n(
volume_to_sell_current_trade
)
self.mc_model.ob.change_volume(
level=self.mc_model.ob.bid,
absolute_level=True,
volume=volume_to_sell_current_trade,
)
level -= 1
self.X_t += trade_turnover
self.Q_t -= volume_to_sell_current_trade
if self.debug:
print(
f"Selling {volume_to_sell_current_trade} for {trade_turnover}, "
f"remaining inventory {self.Q_t}"
)
elif self.Q_t < 0:
volume_to_buy_current_trade = np.min(
[-self.Q_t, self.mc_model.ob.ask_volume]
)
if (
self.mc_model.ob.ask_volume == 0
and self.mc_model.ob.outside_volume != 0
):
volume_to_buy_current_trade = self.mc_model.ob.outside_volume
elif (
self.mc_model.ob.ask_volume == 0
and self.mc_model.ob.outside_volume == 0
):
can_trade = False
if can_trade:
if np.sum(self.mc_model.ob.q_ask()) == 0:
trade_turnover = level * self.mc_model.ob.outside_volume
else:
trade_turnover = self.mc_model.ob.buy_n(
volume_to_buy_current_trade
)
self.mc_model.ob.change_volume(
level=self.mc_model.ob.ask,
absolute_level=True,
volume=-volume_to_buy_current_trade,
)
level += 1
self.X_t -= trade_turnover
self.Q_t += volume_to_buy_current_trade
if self.debug:
print(
f"Buying {volume_to_buy_current_trade} for {trade_turnover}, "
f"remaining inventory {self.Q_t}"
)
else:
can_trade = False
if self.debug:
print("Liquidation done!")
print(f"The bid volumes: {self.mc_model.ob.q_bid()}")
print(f"The ask volumes: {self.mc_model.ob.q_ask()}")
print(f"MM's inventory after liquidation: {self.Q_t}")
# --------------- Simulating the environment ---------------
self.t += self.dt # increase the current time stamp
# Simulating the LOB
if self.t >= self.T:
simulation_results = self.mc_model.simulate(end_time=self.dt)
else:
simulation_results = self.mc_model.simulate(
end_time=self.dt,
order_volumes=self.order_volumes,
order_prices=self.quotes_absolute_level,
)
# We are only interested in the actual simulation results before the terminal time step since we have no
# outstanding limit orders by then
if self.t < self.T:
# Market maker's absolute price levels
mm_bid_abs = self.quotes_absolute_level["bid"]
mm_ask_abs = self.quotes_absolute_level["ask"]
# Looping through all simulated results
for n in range(simulation_results["num_events"]):
event = simulation_results["event"][n]
absolute_level = simulation_results["abs_level"][n]
size = simulation_results["size"][n]
# Arriving LO buy or LO buy cancellation
if event in [0, 4] and absolute_level == mm_bid_abs:
if event == 0: # LO buy
vol_on_mm_bid += size
else: # LO buy cancellation
vol_on_mm_bid -= size
# Arriving LO sell or LO sell cancellation
elif event in [1, 5] and absolute_level == mm_ask_abs:
if event == 1: # LO sell
vol_on_mm_ask += size
else: # LO sell cancellation
vol_on_mm_ask -= size
# MM's orders only affected by MOs
elif event in [2, 3] and self.t != self.T:
# event type 2: 'mo bid', i.e., MO sell order arrives
if (
event == 2
and absolute_level == self.quotes_absolute_level["bid"]
):
if not self.mm_priority:
# IMPLICIT ASSUMPTION THAT MARKET MAKER HAS LAST ORDER PRIORITY
if vol_on_mm_bid - size < self.order_volumes["bid"]:
mm_trade_volume = size - (
vol_on_mm_bid - self.order_volumes["bid"]
)
self.order_volumes[
"bid"
] -= mm_trade_volume # decrease outstanding LO volume
self.X_t -= (
mm_trade_volume * absolute_level
) # deduct cash
self.Q_t += mm_trade_volume # adjust inventory
else:
# IMPLICIT ASSUMPTION THAT MARKET MAKER HAS FIRST ORDER PRIORITY
mm_trade_volume = np.min([size, self.order_volumes["bid"]])
self.order_volumes["bid"] -= mm_trade_volume
self.X_t -= mm_trade_volume * absolute_level
self.Q_t += mm_trade_volume
vol_on_mm_bid -= size
# event type 3: 'mo ask', i.e., MO buy order arrives
elif (
event == 3
and absolute_level == self.quotes_absolute_level["ask"]
):
if not self.mm_priority:
# IMPLICIT ASSUMPTION THAT MARKET MAKER HAS LAST ORDER PRIORITY
if vol_on_mm_ask - size < self.order_volumes["ask"]:
mm_trade_volume = size - (
vol_on_mm_ask - self.order_volumes["ask"]
)
self.order_volumes[
"ask"
] -= mm_trade_volume # decrease outstanding LO volume
self.X_t += (
mm_trade_volume * absolute_level
) # increase cash
self.Q_t -= mm_trade_volume # adjust inventory
else:
# IMPLICIT ASSUMPTION THAT MARKET MAKER HAS FIRST ORDER PRIORITY
mm_trade_volume = np.min([size, self.order_volumes["ask"]])
self.order_volumes["ask"] -= mm_trade_volume
self.X_t += mm_trade_volume * absolute_level
self.Q_t -= mm_trade_volume
vol_on_mm_ask -= size
if self.debug:
if (
self.X_t
!= self.print_MO_results(
simulation_results, self.X_t_previous, printing=False
)
and self.t < self.T
):
self.num_errors += 1
self.print_MO_results(
simulation_results, self.X_t_previous, printing=True
)
print("--> should have been", self.X_t, "at time", self.t)
input("Next:")
# If the market maker has an outstanding buy order, cancel the entire volume
if self.t != self.T:
if self.order_volumes["bid"] > 0:
self.mc_model.ob.change_volume(
level=self.quotes_absolute_level["bid"],
volume=np.min(
[
self.order_volumes["bid"],
-self.mc_model.ob.get_volume(
self.quotes_absolute_level["bid"], absolute_level=True
),
]
),
absolute_level=True,
)
# If the market maker has an outstanding sell order, cancel the entire volume
if self.order_volumes["ask"] > 0:
self.mc_model.ob.change_volume(
level=self.quotes_absolute_level["ask"],
volume=-np.min(
[
self.order_volumes["ask"],
self.mc_model.ob.get_volume(
self.quotes_absolute_level["ask"], absolute_level=True
),
]
),
absolute_level=True,
)
if self.Q_t > 0:
self.H_t = self.mc_model.ob.sell_n(self.Q_t)
else:
self.H_t = -self.mc_model.ob.buy_n(-self.Q_t)
return self.state(), self._get_reward(), self.t == self.T, {}
def _get_reward(self):
"""
Returns the reward
Parameters
----------
None
Returns
-------
reward : float
the reward
"""
return self.reward_scale * (
self.X_t
+ self.H_t
- (self.X_t_previous + self.H_t_previous)
- self.phi * self.Q_t**2
)
def pre_run(self, n_steps=int(1e4)):
"""
Simulates n_steps events / transitions in the Markov chain LOB
Parameters
----------
n_steps : int
the number of events that should be simulated in the LOB
Returns
-------
None
"""
self.mc_model.simulate(int(n_steps))
def reset(self, randomized=None):
"""
Reset the environment. Chooses a random LOB state if prompted.
Parameters
----------
randmoized : bool
if the LOB state should be randomized
Returns
-------
None
"""
self.X_t = 0
self.t = 0
self.Q_t = 0
# Ugly but for efficiency
if randomized == None and self.randomize_reset:
ob_start = random.choice(self.starts_database)
elif randomized:
ob_start = random.choice(self.starts_database)
else:
ob_start = self.starts_database[0]
self.mc_model = MarkovChainLobModel(
include_spread_levels=self.include_spread_levels,
ob_start=ob_start,
num_levels=self.num_levels,
)
if self.pre_run_on_start:
self.pre_run()
return self.state()
def render(self, mode="human"):
"""
Prints useful stats of the environment
Parameters
----------
mode : str
??
Returns
-------
None
"""
print("=" * 40)
print(f"End of t = {self.t}")
print(f"Current bid-ask = {self.mc_model.ob.bid}-{self.mc_model.ob.ask}")
print(f"Current inventory = {self.Q_t}")
print(f"Cash value = {self.X_t}")
print(f"Holding value = {self.H_t}")
print(f"Total value = {self.X_t + self.H_t}")
print(f"State = {self.state()}")
print("=" * 40 + "\n")
def print_simulation_results(self, simulation_results):
"""
Prints the events after a simulation
Parameters
----------
simulation_results : dict
a dictionary with events
Returns
-------
None
"""
for key in simulation_results.keys():
if key == "event":
print("\n event : ", end="")
for event in simulation_results[key]:
print(self.mc_model.inverse_event_types[event], end=", ")
else:
print("\n", key, ":", simulation_results[key], end="")
def print_MO_results(self, simulation_results, cash, printing=True):
"""
Prints the MOs after a simulation. Used for debugging.
Parameters
----------
simulation_results : dict
a dictionary with events
Returns
-------
None
"""
if 2 in simulation_results["event"] or 3 in simulation_results["event"]:
if printing:
print("=" * 40)
self.print_simulation_results(simulation_results)
print("\n" + "-" * 40)
print("t =", self.t)
print(self.order_volumes)
print(self.quotes_absolute_level)
size_bid = self.default_order_size
size_ask = self.default_order_size
for i, event in enumerate(simulation_results["event"]):
size_bid_event = min(simulation_results["size"][i], size_bid)
size_ask_event = min(simulation_results["size"][i], size_ask)
level = simulation_results["abs_level"][i]
if (
self.mc_model.inverse_event_types[event] == "mo bid"
and level == self.quotes_absolute_level["bid"]
):
if printing:
print("MO sell, size", str(size_bid_event) + ", price", level)
print(
"cash:",
cash,
"-->",
cash,
"-",
size_bid_event,
"*",
level,
"=",
cash - size_bid_event * level,
)
cash -= size_bid_event * level
size_bid -= size_bid_event
elif (
self.mc_model.inverse_event_types[event] == "mo ask"
and level == self.quotes_absolute_level["ask"]
):
if printing:
print("MO buy, size", str(size_ask_event) + ", price", level)
print(
"cash:",
cash,
"-->",
cash,
"+",
size_ask_event,
"*",
level,
"=",
cash + size_ask_event * level,
)
cash += size_ask_event * level
size_ask -= size_ask_event
if printing:
print("=" * 40)
return cash