From 86c4c27a6a538d3f5ccbf4ca8765a49d90ef54a4 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 14 Nov 2016 17:35:29 -0500 Subject: [PATCH 01/13] MAINT: Factored out order arg calculation methods so callers can use them to construct args for a batch of orders --- zipline/algorithm.py | 63 +++++++++++++++++++++++--------------- zipline/finance/blotter.py | 2 +- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index 2b6facc41e..c2052283f4 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -1404,6 +1404,12 @@ def order(self, if not self._can_order_asset(asset): return None + amount, style = self._calculate_order(asset, amount, + limit_price, stop_price, style) + return self.blotter.order(asset, amount, style) + + def _calculate_order(self, asset, amount, + limit_price=None, stop_price=None, style=None): # Truncate to the integer share count that's either within .0001 of # amount or closer to zero. # E.g. 3.9999 -> 4.0; 5.5 -> 5.0; -5.5 -> -5.0 @@ -1421,7 +1427,7 @@ def order(self, style = self.__convert_order_params_for_blotter(limit_price, stop_price, style) - return self.blotter.order(asset, amount, style) + return amount, style def validate_order_params(self, asset, @@ -1744,11 +1750,15 @@ def order_percent(self, if not self._can_order_asset(asset): return None + amount = self._calculate_order_percent_amount(asset, percent) + return self.order(asset, amount, + limit_price=limit_price, + stop_price=stop_price, + style=style) + + def _calculate_order_percent_amount(self, asset, percent): value = self.portfolio.portfolio_value * percent - return self.order_value(asset, value, - limit_price=limit_price, - stop_price=stop_price, - style=style) + return self._calculate_order_value_amount(asset, value) @api_method @disallowed_in_before_trading_start(OrderInBeforeTradingStart()) @@ -1810,18 +1820,18 @@ def order_target(self, if not self._can_order_asset(asset): return None + amount = self._calculate_order_target_amount(asset, target) + return self.order(asset, amount, + limit_price=limit_price, + stop_price=stop_price, + style=style) + + def _calculate_order_target_amount(self, asset, target): if asset in self.portfolio.positions: current_position = self.portfolio.positions[asset].amount - req_shares = target - current_position - return self.order(asset, req_shares, - limit_price=limit_price, - stop_price=stop_price, - style=style) - else: - return self.order(asset, target, - limit_price=limit_price, - stop_price=stop_price, - style=style) + target -= current_position + + return target @api_method @disallowed_in_before_trading_start(OrderInBeforeTradingStart()) @@ -1885,10 +1895,11 @@ def order_target_value(self, return None target_amount = self._calculate_order_value_amount(asset, target) - return self.order_target(asset, target_amount, - limit_price=limit_price, - stop_price=stop_price, - style=style) + amount = self._calculate_order_target_amount(asset, target_amount) + return self.order(asset, amount, + limit_price=limit_price, + stop_price=stop_price, + style=style) @api_method @disallowed_in_before_trading_start(OrderInBeforeTradingStart()) @@ -1947,11 +1958,15 @@ def order_target_percent(self, asset, target, if not self._can_order_asset(asset): return None - target_value = self.portfolio.portfolio_value * target - return self.order_target_value(asset, target_value, - limit_price=limit_price, - stop_price=stop_price, - style=style) + amount = self._calculate_order_target_percent_amount(asset, target) + return self.order(asset, amount, + limit_price=limit_price, + stop_price=stop_price, + style=style) + + def _calculate_order_target_percent_amount(self, asset, target): + target_amount = self._calculate_order_percent_amount(asset, target) + return self._calculate_order_target_amount(asset, target_amount) @error_keywords(sid='Keyword argument `sid` is no longer supported for ' 'get_open_orders. Use `asset` instead.') diff --git a/zipline/finance/blotter.py b/zipline/finance/blotter.py index d6c3e37775..e0d26695f5 100644 --- a/zipline/finance/blotter.py +++ b/zipline/finance/blotter.py @@ -91,7 +91,7 @@ def order(self, sid, amount, style, order_id=None): """ if amount == 0: # Don't bother placing orders for 0 shares. - return + return None elif amount > self.max_shares: # Arbitrary limit of 100 billion (US) shares will never be # exceeded except by a buggy algorithm. From 321e52481c75b368b8335e1ab1879a230af2ee3f Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 14 Nov 2016 17:36:15 -0500 Subject: [PATCH 02/13] ENH: Blotter support for ordering a batch --- zipline/finance/blotter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zipline/finance/blotter.py b/zipline/finance/blotter.py index e0d26695f5..99e50eb4e3 100644 --- a/zipline/finance/blotter.py +++ b/zipline/finance/blotter.py @@ -114,6 +114,9 @@ def order(self, sid, amount, style, order_id=None): return order.id + def order_batch(self, orders): + return [self.order(*order) for order in orders] + def cancel(self, order_id, relay_status=True): if order_id not in self.orders: return From 6c04f30eca7e36803831882642b23e2b810d7579 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 14 Nov 2016 17:36:59 -0500 Subject: [PATCH 03/13] MAINT: Removed unnecessary override --- tests/pipeline/test_quarters_estimates.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/pipeline/test_quarters_estimates.py b/tests/pipeline/test_quarters_estimates.py index 52cd08e1ce..5fcd4dee66 100644 --- a/tests/pipeline/test_quarters_estimates.py +++ b/tests/pipeline/test_quarters_estimates.py @@ -453,10 +453,6 @@ def create_estimates_df(cls, SID_FIELD_NAME: sid, }) - @classmethod - def init_class_fixtures(cls): - super(WithEstimatesTimeZero, cls).init_class_fixtures() - def get_expected_estimate(self, q1_knowledge, q2_knowledge, From 3fd34127c016b81a96a4eb66ded0be1cf88e6de0 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 15 Nov 2016 11:15:24 -0500 Subject: [PATCH 04/13] MAINT: Moved common asset lookup to fixture init Also can use class's asset_finder instead of via env --- tests/test_blotter.py | 113 ++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 60 deletions(-) diff --git a/tests/test_blotter.py b/tests/test_blotter.py index fa5d0cf30e..690623e373 100644 --- a/tests/test_blotter.py +++ b/tests/test_blotter.py @@ -50,6 +50,12 @@ class BlotterTestCase(WithCreateBarData, END_DATE = pd.Timestamp('2006-01-06', tz='utc') ASSET_FINDER_EQUITY_SIDS = 24, 25 + @classmethod + def init_class_fixtures(cls): + super(BlotterTestCase, cls).init_class_fixtures() + cls.asset_24 = cls.asset_finder.retrieve_asset(24) + cls.asset_25 = cls.asset_finder.retrieve_asset(25) + @classmethod def make_equity_daily_bar_data(cls): yield 24, pd.DataFrame( @@ -83,64 +89,59 @@ def CREATE_BARDATA_DATA_FREQUENCY(cls): (StopLimitOrder(10, 20), 10, 20)]) def test_blotter_order_types(self, style_obj, expected_lmt, expected_stp): - blotter = Blotter('daily', self.env.asset_finder) + blotter = Blotter('daily', self.asset_finder) - asset_24 = blotter.asset_finder.retrieve_asset(24) - blotter.order(asset_24, 100, style_obj) - result = blotter.open_orders[asset_24][0] + blotter.order(self.asset_24, 100, style_obj) + result = blotter.open_orders[self.asset_24][0] self.assertEqual(result.limit, expected_lmt) self.assertEqual(result.stop, expected_stp) def test_cancel(self): - blotter = Blotter('daily', self.env.asset_finder) - - asset_24 = blotter.asset_finder.retrieve_asset(24) - asset_25 = blotter.asset_finder.retrieve_asset(25) + blotter = Blotter('daily', self.asset_finder) - oid_1 = blotter.order(asset_24, 100, MarketOrder()) - oid_2 = blotter.order(asset_24, 200, MarketOrder()) - oid_3 = blotter.order(asset_24, 300, MarketOrder()) + oid_1 = blotter.order(self.asset_24, 100, MarketOrder()) + oid_2 = blotter.order(self.asset_24, 200, MarketOrder()) + oid_3 = blotter.order(self.asset_24, 300, MarketOrder()) # Create an order for another asset to verify that we don't remove it # when we do cancel_all on 24. - blotter.order(asset_25, 150, MarketOrder()) + blotter.order(self.asset_25, 150, MarketOrder()) self.assertEqual(len(blotter.open_orders), 2) - self.assertEqual(len(blotter.open_orders[asset_24]), 3) + self.assertEqual(len(blotter.open_orders[self.asset_24]), 3) self.assertEqual( - [o.amount for o in blotter.open_orders[asset_24]], + [o.amount for o in blotter.open_orders[self.asset_24]], [100, 200, 300], ) blotter.cancel(oid_2) self.assertEqual(len(blotter.open_orders), 2) - self.assertEqual(len(blotter.open_orders[asset_24]), 2) + self.assertEqual(len(blotter.open_orders[self.asset_24]), 2) self.assertEqual( - [o.amount for o in blotter.open_orders[asset_24]], + [o.amount for o in blotter.open_orders[self.asset_24]], [100, 300], ) self.assertEqual( - [o.id for o in blotter.open_orders[asset_24]], + [o.id for o in blotter.open_orders[self.asset_24]], [oid_1, oid_3], ) - blotter.cancel_all_orders_for_asset(asset_24) + blotter.cancel_all_orders_for_asset(self.asset_24) self.assertEqual(len(blotter.open_orders), 1) - self.assertEqual(list(blotter.open_orders), [asset_25]) + self.assertEqual(list(blotter.open_orders), [self.asset_25]) def test_blotter_eod_cancellation(self): - blotter = Blotter('minute', self.env.asset_finder, + blotter = Blotter('minute', self.asset_finder, cancel_policy=EODCancel()) - asset_24 = blotter.asset_finder.retrieve_asset(24) # Make two orders for the same sid, so we can test that we are not # mutating the orders list as we are cancelling orders - blotter.order(asset_24, 100, MarketOrder()) - blotter.order(asset_24, -100, MarketOrder()) + blotter.order(self.asset_24, 100, MarketOrder()) + blotter.order(self.asset_24, -100, MarketOrder()) self.assertEqual(len(blotter.new_orders), 2) - order_ids = [order.id for order in blotter.open_orders[asset_24]] + order_ids = [order.id for order in blotter.open_orders[self.asset_24]] self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN) self.assertEqual(blotter.new_orders[1].status, ORDER_STATUS.OPEN) @@ -155,11 +156,10 @@ def test_blotter_eod_cancellation(self): self.assertEqual(order.status, ORDER_STATUS.CANCELLED) def test_blotter_never_cancel(self): - blotter = Blotter('minute', self.env.asset_finder, + blotter = Blotter('minute', self.asset_finder, cancel_policy=NeverCancel()) - blotter.order(blotter.asset_finder.retrieve_asset(24), 100, - MarketOrder()) + blotter.order(self.asset_24, 100, MarketOrder()) self.assertEqual(len(blotter.new_orders), 1) self.assertEqual(blotter.new_orders[0].status, ORDER_STATUS.OPEN) @@ -172,8 +172,7 @@ def test_blotter_never_cancel(self): def test_order_rejection(self): blotter = Blotter(self.sim_params.data_frequency, - self.env.asset_finder) - asset_24 = blotter.asset_finder.retrieve_asset(24) + self.asset_finder) # Reject a nonexistent order -> no order appears in new_order, # no exceptions raised out @@ -181,10 +180,10 @@ def test_order_rejection(self): self.assertEqual(blotter.new_orders, []) # Basic tests of open order behavior - open_order_id = blotter.order(asset_24, 100, MarketOrder()) - second_order_id = blotter.order(asset_24, 50, MarketOrder()) - self.assertEqual(len(blotter.open_orders[asset_24]), 2) - open_order = blotter.open_orders[asset_24][0] + open_order_id = blotter.order(self.asset_24, 100, MarketOrder()) + second_order_id = blotter.order(self.asset_24, 50, MarketOrder()) + self.assertEqual(len(blotter.open_orders[self.asset_24]), 2) + open_order = blotter.open_orders[self.asset_24][0] self.assertEqual(open_order.status, ORDER_STATUS.OPEN) self.assertEqual(open_order.id, open_order_id) self.assertIn(open_order, blotter.new_orders) @@ -192,7 +191,7 @@ def test_order_rejection(self): # Reject that order immediately (same bar, i.e. still in new_orders) blotter.reject(open_order_id) self.assertEqual(len(blotter.new_orders), 2) - self.assertEqual(len(blotter.open_orders[asset_24]), 1) + self.assertEqual(len(blotter.open_orders[self.asset_24]), 1) still_open_order = blotter.new_orders[0] self.assertEqual(still_open_order.id, second_order_id) self.assertEqual(still_open_order.status, ORDER_STATUS.OPEN) @@ -203,9 +202,9 @@ def test_order_rejection(self): # Do it again, but reject it at a later time (after tradesimulation # pulls it from new_orders) blotter = Blotter(self.sim_params.data_frequency, - self.env.asset_finder) - new_open_id = blotter.order(asset_24, 10, MarketOrder()) - new_open_order = blotter.open_orders[asset_24][0] + self.asset_finder) + new_open_id = blotter.order(self.asset_24, 10, MarketOrder()) + new_open_order = blotter.open_orders[self.asset_24][0] self.assertEqual(new_open_id, new_open_order.id) # Pretend that the trade simulation did this. blotter.new_orders = [] @@ -220,9 +219,9 @@ def test_order_rejection(self): # You can't reject a filled order. # Reset for paranoia blotter = Blotter(self.sim_params.data_frequency, - self.env.asset_finder) + self.asset_finder) blotter.slippage_func = FixedSlippage() - filled_id = blotter.order(asset_24, 100, MarketOrder()) + filled_id = blotter.order(self.asset_24, 100, MarketOrder()) filled_order = None blotter.current_dt = self.sim_params.sessions[-1] bar_data = self.create_bardata( @@ -236,7 +235,7 @@ def test_order_rejection(self): self.assertEqual(filled_order.id, filled_id) self.assertIn(filled_order, blotter.new_orders) self.assertEqual(filled_order.status, ORDER_STATUS.FILLED) - self.assertNotIn(filled_order, blotter.open_orders[asset_24]) + self.assertNotIn(filled_order, blotter.open_orders[self.asset_24]) blotter.reject(filled_id) updated_order = blotter.orders[filled_id] @@ -249,27 +248,25 @@ def test_order_hold(self): status to OPEN/FILLED as necessary """ blotter = Blotter(self.sim_params.data_frequency, - self.env.asset_finder) + self.asset_finder) # Nothing happens on held of a non-existent order blotter.hold(56) self.assertEqual(blotter.new_orders, []) - asset_24 = blotter.asset_finder.retrieve_asset(24) - - open_id = blotter.order(asset_24, 100, MarketOrder()) - open_order = blotter.open_orders[asset_24][0] + open_id = blotter.order(self.asset_24, 100, MarketOrder()) + open_order = blotter.open_orders[self.asset_24][0] self.assertEqual(open_order.id, open_id) blotter.hold(open_id) self.assertEqual(len(blotter.new_orders), 1) - self.assertEqual(len(blotter.open_orders[asset_24]), 1) + self.assertEqual(len(blotter.open_orders[self.asset_24]), 1) held_order = blotter.new_orders[0] self.assertEqual(held_order.status, ORDER_STATUS.HELD) self.assertEqual(held_order.reason, '') blotter.cancel(held_order.id) self.assertEqual(len(blotter.new_orders), 1) - self.assertEqual(len(blotter.open_orders[asset_24]), 0) + self.assertEqual(len(blotter.open_orders[self.asset_24]), 0) cancelled_order = blotter.new_orders[0] self.assertEqual(cancelled_order.id, held_order.id) self.assertEqual(cancelled_order.status, ORDER_STATUS.CANCELLED) @@ -288,10 +285,9 @@ def test_order_hold(self): ORDER_STATUS.FILLED blotter = Blotter(self.sim_params.data_frequency, - self.env.asset_finder) - open_id = blotter.order(blotter.asset_finder.retrieve_asset(24), - order_size, MarketOrder()) - open_order = blotter.open_orders[asset_24][0] + self.asset_finder) + open_id = blotter.order(self.asset_24, order_size, MarketOrder()) + open_order = blotter.open_orders[self.asset_24][0] self.assertEqual(open_id, open_order.id) blotter.hold(open_id) held_order = blotter.new_orders[0] @@ -312,26 +308,23 @@ def test_order_hold(self): def test_prune_orders(self): blotter = Blotter(self.sim_params.data_frequency, - self.env.asset_finder) - - asset_24 = blotter.asset_finder.retrieve_asset(24) - asset_25 = blotter.asset_finder.retrieve_asset(25) + self.asset_finder) - blotter.order(asset_24, 100, MarketOrder()) - open_order = blotter.open_orders[asset_24][0] + blotter.order(self.asset_24, 100, MarketOrder()) + open_order = blotter.open_orders[self.asset_24][0] blotter.prune_orders([]) - self.assertEqual(1, len(blotter.open_orders[asset_24])) + self.assertEqual(1, len(blotter.open_orders[self.asset_24])) blotter.prune_orders([open_order]) - self.assertEqual(0, len(blotter.open_orders[asset_24])) + self.assertEqual(0, len(blotter.open_orders[self.asset_24])) # prune an order that isn't in our our open orders list, make sure # nothing blows up other_order = Order( dt=blotter.current_dt, - sid=asset_25, + sid=self.asset_25, amount=1 ) From f4773053cb9929be0bdc0268b50a8f506796764d Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 15 Nov 2016 11:16:28 -0500 Subject: [PATCH 05/13] TST: Added test for order_batch --- tests/test_blotter.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_blotter.py b/tests/test_blotter.py index 690623e373..86fba88a71 100644 --- a/tests/test_blotter.py +++ b/tests/test_blotter.py @@ -329,3 +329,37 @@ def test_prune_orders(self): ) blotter.prune_orders([other_order]) + + def test_order_batch_matches_multi_order(self): + """ + Ensure the effect of order_batch is the same as multiple calls to + order. + """ + blotter1 = Blotter(self.sim_params.data_frequency, + self.asset_finder) + blotter2 = Blotter(self.sim_params.data_frequency, + self.asset_finder) + for i in range(1, 4): + order_args = [ + (self.asset_24, i * 100, MarketOrder()), + (self.asset_25, i * 100, LimitOrder(i * 100 + 1)), + ] + + order_batch_ids = blotter1.order_batch(order_args) + order_ids = [] + for order_arg in order_args: + order_ids.append(blotter2.order(*order_arg)) + self.assertEqual(len(order_batch_ids), len(order_ids)) + + self.assertEqual(len(blotter1.open_orders), + len(blotter2.open_orders)) + + for (asset, _, _), order_batch_id, order_id in zip( + order_args, order_batch_ids, order_ids + ): + self.assertEqual(len(blotter1.open_orders[asset]), + len(blotter2.open_orders[asset])) + self.assertEqual(order_batch_id, + blotter1.open_orders[asset][i-1].id) + self.assertEqual(order_id, + blotter2.open_orders[asset][i-1].id) From 7aefa9c3110a34bb9b1cacf8652bcd642849f820 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 17 Nov 2016 18:12:29 -0500 Subject: [PATCH 06/13] MAINT: Allow for orders with id 0 --- zipline/finance/order.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zipline/finance/order.py b/zipline/finance/order.py index 39e3996a1a..d326f206f4 100644 --- a/zipline/finance/order.py +++ b/zipline/finance/order.py @@ -58,7 +58,7 @@ def __init__(self, dt, sid, amount, stop=None, limit=None, filled=0, assert isinstance(sid, Asset) # get a string representation of the uuid. - self.id = id or self.make_id() + self.id = self.make_id() if id is None else id self.dt = dt self.reason = None self.created = dt From 74a3247892fc047dd356e14f90c727c2627a41e2 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 23 Nov 2016 17:49:39 -0500 Subject: [PATCH 07/13] MAINT: Renamed order_batch parameter and added docs --- tests/test_blotter.py | 10 ++++---- zipline/algorithm.py | 5 ++-- zipline/finance/blotter.py | 51 +++++++++++++++++++++++++++++++++----- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/tests/test_blotter.py b/tests/test_blotter.py index 86fba88a71..deb0bbdb50 100644 --- a/tests/test_blotter.py +++ b/tests/test_blotter.py @@ -340,22 +340,22 @@ def test_order_batch_matches_multi_order(self): blotter2 = Blotter(self.sim_params.data_frequency, self.asset_finder) for i in range(1, 4): - order_args = [ + order_arg_lists = [ (self.asset_24, i * 100, MarketOrder()), (self.asset_25, i * 100, LimitOrder(i * 100 + 1)), ] - order_batch_ids = blotter1.order_batch(order_args) + order_batch_ids = blotter1.order_batch(order_arg_lists) order_ids = [] - for order_arg in order_args: - order_ids.append(blotter2.order(*order_arg)) + for order_args in order_arg_lists: + order_ids.append(blotter2.order(*order_args)) self.assertEqual(len(order_batch_ids), len(order_ids)) self.assertEqual(len(blotter1.open_orders), len(blotter2.open_orders)) for (asset, _, _), order_batch_id, order_id in zip( - order_args, order_batch_ids, order_ids + order_arg_lists, order_batch_ids, order_ids ): self.assertEqual(len(blotter1.open_orders[asset]), len(blotter2.open_orders[asset])) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index c2052283f4..2cfd4bf9fa 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -1382,8 +1382,9 @@ def order(self, Returns ------- - order_id : str - The unique identifier for this order. + order_id : str or None + The unique identifier for this order, or None if no order was + placed. Notes ----- diff --git a/zipline/finance/blotter.py b/zipline/finance/blotter.py index 99e50eb4e3..fc1b3558dc 100644 --- a/zipline/finance/blotter.py +++ b/zipline/finance/blotter.py @@ -75,12 +75,29 @@ def set_date(self, dt): self.current_dt = dt def order(self, sid, amount, style, order_id=None): + """Place an order. - # something could be done with amount to further divide - # between buy by share count OR buy shares up to a dollar amount - # numeric == share count AND "$dollar.cents" == cost amount + Parameters + ---------- + asset : zipline.assets.Asset + The asset that this order is for. + amount : int + The amount of shares to order. If ``amount`` is positive, this is + the number of shares to buy or cover. If ``amount`` is negative, + this is the number of shares to sell or short. + style : zipline.finance.execution.ExecutionStyle + The execution style for the order. + order_id : str, optional + The unique identifier for this order. - """ + Returns + ------- + order_id : str or None + The unique identifier for this order, or None if no order was + placed. + + Notes + ----- amount > 0 :: Buy/Cover amount < 0 :: Sell/Short Market order: order(sid, amount) @@ -89,6 +106,10 @@ def order(self, sid, amount, style, order_id=None): StopLimit order: order(sid, amount, style=StopLimitOrder(limit_price, stop_price)) """ + # something could be done with amount to further divide + # between buy by share count OR buy shares up to a dollar amount + # numeric == share count AND "$dollar.cents" == cost amount + if amount == 0: # Don't bother placing orders for 0 shares. return None @@ -114,8 +135,26 @@ def order(self, sid, amount, style, order_id=None): return order.id - def order_batch(self, orders): - return [self.order(*order) for order in orders] + def order_batch(self, order_arg_lists): + """Place a batch of orders. + + Parameters + ---------- + order_arg_lists : iterable[tuple] + Tuples of args that `order` expects. + + Returns + ------- + order_ids : list[str or None] + The unique identifier (or None) for each of the orders placed + (or not placed). + + Notes + ----- + This is required for `Blotter` subclasses to be able to place a batch + of orders, instead of being passed the order requests one at a time. + """ + return [self.order(*order_args) for order_args in order_arg_lists] def cancel(self, order_id, relay_status=True): if order_id not in self.orders: From 8ea3226a5cf13a2f1dbaf8a3d4706528171914b6 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 28 Nov 2016 18:37:03 -0500 Subject: [PATCH 08/13] ENH: Renamed to batch_order and added batch_order_target_percent --- tests/test_algorithm.py | 16 +++++++++------- tests/test_blotter.py | 4 ++-- zipline/algorithm.py | 38 +++++++++++++++++++++++++++++++++++++- zipline/finance/blotter.py | 2 +- zipline/test_algorithms.py | 10 +++++++++- 5 files changed, 58 insertions(+), 12 deletions(-) diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index e5c4e2edd1..85a7a65fcc 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -123,6 +123,7 @@ TestTargetAlgorithm, TestTargetPercentAlgorithm, TestTargetValueAlgorithm, + TestBatchTargetPercentAlgorithm, SetLongOnlyAlgorithm, SetAssetDateBoundsAlgorithm, SetMaxPositionSizeAlgorithm, @@ -905,14 +906,15 @@ def test_data_frequency_setting(self): self.assertEqual(algo.sim_params.data_frequency, 'minute') @parameterized.expand([ - (TestOrderAlgorithm,), - (TestOrderValueAlgorithm,), - (TestTargetAlgorithm,), - (TestOrderPercentAlgorithm,), - (TestTargetPercentAlgorithm,), - (TestTargetValueAlgorithm,), + ('order', TestOrderAlgorithm,), + ('order_value', TestOrderValueAlgorithm,), + ('order_target', TestTargetAlgorithm,), + ('order_percent', TestOrderPercentAlgorithm,), + ('order_target_percent', TestTargetPercentAlgorithm,), + ('order_target_value', TestTargetValueAlgorithm,), + ('batch_order_target_percent', TestBatchTargetPercentAlgorithm,), ]) - def test_order_methods(self, algo_class): + def test_order_methods(self, test_name, algo_class): algo = algo_class( sim_params=self.sim_params, env=self.env, diff --git a/tests/test_blotter.py b/tests/test_blotter.py index deb0bbdb50..2126095414 100644 --- a/tests/test_blotter.py +++ b/tests/test_blotter.py @@ -330,7 +330,7 @@ def test_prune_orders(self): blotter.prune_orders([other_order]) - def test_order_batch_matches_multi_order(self): + def test_batch_order_matches_multiple_orders(self): """ Ensure the effect of order_batch is the same as multiple calls to order. @@ -345,7 +345,7 @@ def test_order_batch_matches_multi_order(self): (self.asset_25, i * 100, LimitOrder(i * 100 + 1)), ] - order_batch_ids = blotter1.order_batch(order_arg_lists) + order_batch_ids = blotter1.batch_order(order_arg_lists) order_ids = [] for order_args in order_arg_lists: order_ids.append(blotter2.order(*order_args)) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index 2cfd4bf9fa..b076e71a4a 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -13,6 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. from collections import Iterable +try: + # optional cython based OrderedDict + from cyordereddict import OrderedDict +except ImportError: + from collections import OrderedDict from copy import copy import operator as op import warnings @@ -33,6 +38,7 @@ itervalues, string_types, viewkeys, + viewvalues, ) from zipline._protocol import handle_non_market_minutes @@ -1916,7 +1922,7 @@ def order_target_percent(self, asset, target, ---------- asset : Asset The asset that this order is for. - percent : float + target : float The desired percentage of the porfolio value to allocate to ``asset``. This is specified as a decimal, for example: 0.50 means 50%. @@ -1969,6 +1975,36 @@ def _calculate_order_target_percent_amount(self, asset, target): target_amount = self._calculate_order_percent_amount(asset, target) return self._calculate_order_target_amount(asset, target_amount) + @api_method + @disallowed_in_before_trading_start(OrderInBeforeTradingStart()) + def batch_order_target_percent(self, weights): + """Place orders towards a given portfolio of weights. + + Parameters + ---------- + weights : collections.Mapping[Asset -> float] + + Returns + ------- + order_ids : pd.Series[Asset -> str] + The unique identifiers for the orders that were placed. + + See Also + -------- + :func:`zipline.api.order_target_percent` + """ + order_args = OrderedDict() + for asset, target in iteritems(weights): + if self._can_order_asset(asset): + amount = self._calculate_order_target_percent_amount( + asset, target, + ) + amount, style = self._calculate_order(asset, amount) + order_args[asset] = (asset, amount, style) + + order_ids = self.blotter.batch_order(viewvalues(order_args)) + return pd.Series(data=order_ids, index=order_args) + @error_keywords(sid='Keyword argument `sid` is no longer supported for ' 'get_open_orders. Use `asset` instead.') @api_method diff --git a/zipline/finance/blotter.py b/zipline/finance/blotter.py index fc1b3558dc..a73dda0dd5 100644 --- a/zipline/finance/blotter.py +++ b/zipline/finance/blotter.py @@ -135,7 +135,7 @@ def order(self, sid, amount, style, order_id=None): return order.id - def order_batch(self, order_arg_lists): + def batch_order(self, order_arg_lists): """Place a batch of orders. Parameters diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index 5d112f1276..7fe1713dd7 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -429,9 +429,17 @@ def handle_data(self, data): "Orders not filled at current price." self.sale_price = data.current(sid(0), "price") - self.order_target_percent(self.sid(0), .002) + self._order(self.sid(0), .002) self.ordered = True + def _order(self, asset, target): + return self.order_target_percent(asset, target) + + +class TestBatchTargetPercentAlgorithm(TestTargetPercentAlgorithm): + def _order(self, asset, target): + return self.batch_order_target_percent({asset: target}) + class TestTargetValueAlgorithm(TradingAlgorithm): def initialize(self): From 836da0f97a41327cde2ed8430ea3ead5175eae30 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 1 Dec 2016 18:30:59 -0500 Subject: [PATCH 09/13] Filter out null entries --- zipline/algorithm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index b076e71a4a..699621135b 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -2003,7 +2003,8 @@ def batch_order_target_percent(self, weights): order_args[asset] = (asset, amount, style) order_ids = self.blotter.batch_order(viewvalues(order_args)) - return pd.Series(data=order_ids, index=order_args) + order_ids = pd.Series(data=order_ids, index=order_args) + return order_ids[~order_ids.isnull()] @error_keywords(sid='Keyword argument `sid` is no longer supported for ' 'get_open_orders. Use `asset` instead.') From 381029ac1a77e09a48f4548ccd5bc1f17387bd12 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 8 Dec 2016 18:50:48 -0500 Subject: [PATCH 10/13] DOC: Updated return types in docstrings --- zipline/testing/fixtures.py | 6 ++++++ zipline/utils/calendars/calendar_utils.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/zipline/testing/fixtures.py b/zipline/testing/fixtures.py index fb051f18c7..34a431bf22 100644 --- a/zipline/testing/fixtures.py +++ b/zipline/testing/fixtures.py @@ -366,6 +366,12 @@ def make_asset_finder_db_url(cls): @classmethod def make_asset_finder(cls): + """Returns a new AssetFinder + + Returns + ------- + asset_finder : zipline.assets.AssetFinder + """ return cls.enter_class_context(tmp_asset_finder( url=cls.make_asset_finder_db_url(), equities=cls.make_equity_info(), diff --git a/zipline/utils/calendars/calendar_utils.py b/zipline/utils/calendars/calendar_utils.py index c8e6e4363c..0a0b03b2f2 100644 --- a/zipline/utils/calendars/calendar_utils.py +++ b/zipline/utils/calendars/calendar_utils.py @@ -67,7 +67,7 @@ def get_calendar(self, name): Returns ------- - TradingCalendar + calendar : zipline.utils.calendars.TradingCalendar The desired calendar. """ canonical_name = self.resolve_alias(name) From 9e73c4f44f21ca899c95c53864568d6c6f4fdc9a Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 20 Dec 2016 18:56:40 -0500 Subject: [PATCH 11/13] TST: Ensure batch_order_target_percent orders like order_target_percent --- tests/test_algorithm.py | 71 +++++++++++++++++++++++++++++++++++++ zipline/testing/__init__.py | 1 + zipline/testing/core.py | 15 ++++++++ 3 files changed, 87 insertions(+) diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 85a7a65fcc..309d96a57c 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -98,6 +98,7 @@ to_utc, trades_by_sid_to_dfs, ) +from zipline.testing import RecordBatchBlotter from zipline.testing.fixtures import ( WithDataPortal, WithLogger, @@ -171,6 +172,7 @@ set_benchmark_algo, no_handle_data, ) +from zipline.testing.predicates import assert_equal from zipline.utils.api_support import ZiplineAPI, set_algo_instance from zipline.utils.calendars import get_calendar, register_calendar from zipline.utils.context_tricks import CallbackManager @@ -1737,6 +1739,75 @@ def test_order_methods(self): ) test_algo.run(self.data_portal) + def test_batch_order_target_percent_matches_multi_order(self): + weights = pd.Series([.3, .7]) + + multi_blotter = RecordBatchBlotter(self.SIM_PARAMS_DATA_FREQUENCY, + self.asset_finder) + multi_test_algo = TradingAlgorithm( + script=dedent("""\ + from collections import OrderedDict + from six import iteritems + + from zipline.api import sid, order_target_percent + + + def initialize(context): + context.assets = [sid(0), sid(3)] + context.placed = False + + def handle_data(context, data): + if not context.placed: + for asset, weight in iteritems(OrderedDict(zip( + context.assets, {weights} + ))): + order_target_percent(asset, weight) + + context.placed = True + + """).format(weights=list(weights)), + blotter=multi_blotter, + env=self.env, + ) + multi_stats = multi_test_algo.run(self.data_portal) + self.assertFalse(multi_blotter.order_batch_called) + + batch_blotter = RecordBatchBlotter(self.SIM_PARAMS_DATA_FREQUENCY, + self.asset_finder) + batch_test_algo = TradingAlgorithm( + script=dedent("""\ + from collections import OrderedDict + + from zipline.api import sid, batch_order_target_percent + + + def initialize(context): + context.assets = [sid(0), sid(3)] + context.placed = False + + def handle_data(context, data): + if not context.placed: + batch_order_target_percent(OrderedDict(zip( + context.assets, {weights} + ))) + context.placed = True + + """).format(weights=list(weights)), + blotter=batch_blotter, + env=self.env, + ) + batch_stats = batch_test_algo.run(self.data_portal) + self.assertTrue(batch_blotter.order_batch_called) + + for stats in (multi_stats, batch_stats): + stats.orders = stats.orders.apply( + lambda orders: [toolz.dissoc(o, 'id') for o in orders] + ) + stats.transactions = stats.transactions.apply( + lambda txns: [toolz.dissoc(txn, 'order_id') for txn in txns] + ) + assert_equal(multi_stats, batch_stats) + def test_order_dead_asset(self): # after asset 0 is dead params = SimulationParameters( diff --git a/zipline/testing/__init__.py b/zipline/testing/__init__.py index ee2aad9dda..1a5449b0de 100644 --- a/zipline/testing/__init__.py +++ b/zipline/testing/__init__.py @@ -7,6 +7,7 @@ FetcherDataPortal, MockDailyBarReader, OpenPrice, + RecordBatchBlotter, add_security_data, all_pairs_matching_predicate, all_subindices, diff --git a/zipline/testing/core.py b/zipline/testing/core.py index 24821d938c..faaa2857c9 100644 --- a/zipline/testing/core.py +++ b/zipline/testing/core.py @@ -39,6 +39,7 @@ BcolzDailyBarWriter, SQLiteAdjustmentWriter, ) +from zipline.finance.blotter import Blotter from zipline.finance.trading import TradingEnvironment from zipline.finance.order import ORDER_STATUS from zipline.lib.labelarray import LabelArray @@ -1502,6 +1503,20 @@ def ensure_doctest(f, name=None): return f +class RecordBatchBlotter(Blotter): + """Blotter that tracks how its batch_order method was called. + """ + def __init__(self, data_frequency, asset_finder): + super(RecordBatchBlotter, self).__init__( + data_frequency, asset_finder, + ) + self.order_batch_called = [] + + def batch_order(self, *args, **kwargs): + self.order_batch_called.append((args, kwargs)) + return super(RecordBatchBlotter, self).batch_order(*args, **kwargs) + + #################################### # Shared factors for pipeline tests. #################################### From 9739ae5532f9a904b602b735ad463db411c734fd Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 20 Dec 2016 18:59:07 -0500 Subject: [PATCH 12/13] MAINT: Some cleanup while working on batch ordering --- tests/test_algorithm.py | 6 +++--- zipline/test_algorithms.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 309d96a57c..81e842eeea 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -939,7 +939,7 @@ def test_order_methods_for_future(self, algo_class): sim_params=self.sim_params, env=self.env, ) - # Ensure that the environment's asset 0 is a Future + # Ensure that the environment's asset 3 is a Future asset_to_test = algo.sid(3) self.assertIsInstance(asset_to_test, Future) @@ -1019,7 +1019,7 @@ def test_minute_data(self, algo_class): sim_params = SimulationParameters( start_session=start_session, end_session=period_end, - capital_base=float("1.0e5"), + capital_base=1.0e5, data_frequency='minute', trading_calendar=self.trading_calendar, ) @@ -3701,7 +3701,7 @@ def init_class_fixtures(cls): cls.first_asset_expiration = cls.test_days[2] def make_data(self, auto_close_delta, frequency, - capital_base=float("1.0e5")): + capital_base=1.0e5): asset_info = make_jagged_equity_info( num_assets=3, diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index 7fe1713dd7..aeb9279cb6 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -411,7 +411,7 @@ def initialize(self): def handle_data(self, data): if not self.ordered: - assert 0 not in self.portfolio.positions + assert not self.portfolio.positions else: # Since you can't own fractional shares (at least in this # example), we want to make sure that our target amount is @@ -429,7 +429,7 @@ def handle_data(self, data): "Orders not filled at current price." self.sale_price = data.current(sid(0), "price") - self._order(self.sid(0), .002) + self._order(sid(0), .002) self.ordered = True def _order(self, asset, target): From 872b6a9ae45cb542218ba4914b6bd548ae1a9224 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 20 Dec 2016 19:09:36 -0500 Subject: [PATCH 13/13] MAINT: Filter out null orders --- tests/test_algorithm.py | 42 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 81e842eeea..71df3863c9 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -1787,9 +1787,14 @@ def initialize(context): def handle_data(context, data): if not context.placed: - batch_order_target_percent(OrderedDict(zip( + orders = batch_order_target_percent(OrderedDict(zip( context.assets, {weights} ))) + assert len(orders) == 2, \ + "len(orders) was %s but expected 2" % len(orders) + for o in orders: + assert o is not None, "An order is None" + context.placed = True """).format(weights=list(weights)), @@ -1808,6 +1813,41 @@ def handle_data(context, data): ) assert_equal(multi_stats, batch_stats) + def test_batch_order_target_percent_filters_null_orders(self): + weights = pd.Series([1, 0]) + + batch_blotter = RecordBatchBlotter(self.SIM_PARAMS_DATA_FREQUENCY, + self.asset_finder) + batch_test_algo = TradingAlgorithm( + script=dedent("""\ + from collections import OrderedDict + + from zipline.api import sid, batch_order_target_percent + + + def initialize(context): + context.assets = [sid(0), sid(3)] + context.placed = False + + def handle_data(context, data): + if not context.placed: + orders = batch_order_target_percent(OrderedDict(zip( + context.assets, {weights} + ))) + assert len(orders) == 1, \ + "len(orders) was %s but expected 1" % len(orders) + for o in orders: + assert o is not None, "An order is None" + + context.placed = True + + """).format(weights=list(weights)), + blotter=batch_blotter, + env=self.env, + ) + batch_test_algo.run(self.data_portal) + self.assertTrue(batch_blotter.order_batch_called) + def test_order_dead_asset(self): # after asset 0 is dead params = SimulationParameters(