Adds Rebate and Fee Rates to Shortable Provider (#7840)

* Adds Rebate and Fee Rates to Shortable Provider

If the provider does have this information, the rates are zero.

* Adds Helper Methods to QCAlgorithm

- Improve summaries.
- Express values as rates instead of percentages.
- Adds headers to files to emulate real data

* Clarify the Data Format

* Removes Helper Methods from QCAlgorithm

* Removes `Shortable` and `ShortableQuantity` from Example

Algorithms should prefer getting the information from the `Security.ShortableProvider`.

Algorithm.CSharp/AllShortableSymbolsCoarseSelectionRegressionAlgorithm.cs

@@ -107,20 +107,18 @@ public override void OnData(Slice data)
                 return;
             }
 
-            foreach (var symbol in ActiveSecurities.Keys.OrderBy(symbol => symbol))
+            foreach (var (symbol, security) in ActiveSecurities.Where(kvp => !kvp.Value.Invested).OrderBy(kvp => kvp.Key))
             {
-                if (!Portfolio.ContainsKey(symbol) || !Portfolio[symbol].Invested)
+                var shortableQuantity = security.ShortableProvider.ShortableQuantity(symbol, Time);
+                if (shortableQuantity == null)
                 {
-                    if (!Shortable(symbol))
-                    {
-                        throw new Exception($"Expected {symbol} to be shortable on {Time:yyyy-MM-dd}");
-                    }
-
-                    // Buy at least once into all Symbols. Since daily data will always use
-                    // MOO orders, it makes the testing of liquidating buying into Symbols difficult.
-                    MarketOrder(symbol, -(decimal)ShortableQuantity(symbol));
-                    _lastTradeDate = Time.Date;
+                    throw new Exception($"Expected {symbol} to be shortable on {Time:yyyy-MM-dd}");
                 }
+
+                // Buy at least once into all Symbols. Since daily data will always use
+                // MOO orders, it makes the testing of liquidating buying into Symbols difficult.
+                MarketOrder(symbol, -(decimal)shortableQuantity);
+                _lastTradeDate = Time.Date;
             }
         }
 

Algorithm.CSharp/CustomShortableProviderRegressionAlgorithm.cs

@@ -62,10 +62,24 @@ public override void OnEndOfAlgorithm()
             {
                 throw new Exception($"Quantity of order {_orderId} should be -1001, but was {orderQuantity}");
             }
+            var feeRate = _spy.ShortableProvider.FeeRate(_spy.Symbol, Time);
+            if (feeRate != 0.0025m)
+            {
+                throw new Exception($"Fee rate should be 0.0025, but was {feeRate}");
+            }
+            var rebateRate = _spy.ShortableProvider.RebateRate(_spy.Symbol, Time);
+            if (rebateRate != 0.0507m)
+            {
+                throw new Exception($"Fee rate should be 0.0507, but was {rebateRate}");
+            }
         }
 
         private class CustomSPYShortableProvider : IShortableProvider
         {
+            public decimal FeeRate(Symbol symbol, DateTime localTime) => 0.0025m;
+
+            public decimal RebateRate(Symbol symbol, DateTime localTime) => 0.0507m;
+
             public long? ShortableQuantity(Symbol symbol, DateTime localTime)
             {
                 if (localTime < new DateTime(2013, 10, 5))

Algorithm.Python/AllShortableSymbolsCoarseSelectionRegressionAlgorithm.py

@@ -54,17 +54,19 @@ def OnData(self, data):
         if self.Time.date() == self.lastTradeDate:
             return
 
-        for symbol in sorted(self.ActiveSecurities.Keys, key = lambda x:x.Value):
-            if (not self.Portfolio.ContainsKey(symbol)) or (not self.Portfolio[symbol].Invested):
-                if not self.Shortable(symbol):
-                    raise Exception(f"Expected {symbol} to be shortable on {self.Time.strftime('%Y%m%d')}")
-
-                """
-                Buy at least once into all Symbols. Since daily data will always use
-                MOO orders, it makes the testing of liquidating buying into Symbols difficult.
-                """
-                self.MarketOrder(symbol, -self.ShortableQuantity(symbol))
-                self.lastTradeDate = self.Time.date()
+        for (symbol, security) in {x.Key: x.Value for x in sorted(self.ActiveSecurities, key = lambda kvp:kvp.Key)}.items():
+            if security.Invested:
+                continue
+            shortable_quantity = security.ShortableProvider.ShortableQuantity(symbol, self.Time)
+            if not shortable_quantity:
+                raise Exception(f"Expected {symbol} to be shortable on {self.Time.strftime('%Y%m%d')}")
+
+            """
+            Buy at least once into all Symbols. Since daily data will always use
+            MOO orders, it makes the testing of liquidating buying into Symbols difficult.
+            """
+            self.MarketOrder(symbol, -shortable_quantity)
+            self.lastTradeDate = self.Time.date()
 
     def CoarseSelection(self, coarse):
         shortableSymbols = self.shortableProvider.AllShortableSymbols(self.Time)

Algorithm.Python/CustomShortableProviderRegressionAlgorithm.py

@@ -38,8 +38,19 @@ def OnEndOfAlgorithm(self):
         orderQuantity = self.Transactions.GetOrderById(self.orderId).Quantity
         if orderQuantity != -1001:
             raise Exception("Quantity of order " + str(_orderId) + " should be " + str(-1001)+", but was {orderQuantity}")
+
+        fee_rate = self.spy.ShortableProvider.FeeRate(self.spy.Symbol, self.Time)
+        if fee_rate != 0.0025:
+            raise Exception(f"Fee rate should be 0.0025, but was {fee_rate}")
+        rebate_rate = self.spy.ShortableProvider.RebateRate(self.spy.Symbol, self.Time)
+        if rebate_rate != 0.0507:
+            raise Exception(f"Rebate rate should be 0.0507, but was {rebate_rate}")
 
 class CustomShortableProvider(NullShortableProvider):
+    def FeeRate(self, symbol: Symbol, localTime: DateTime):
+        return 0.0025
+    def RebateRate(self, symbol: Symbol, localTime: DateTime):
+        return 0.0507
     def ShortableQuantity(self, symbol: Symbol, localTime: DateTime):
         if localTime < datetime(2013,10,5):
             return 10

Common/Data/Shortable/LocalDiskShortableProvider.cs

@@ -18,8 +18,8 @@
 using QuantConnect.Util;
 using System.Threading.Tasks;
 using QuantConnect.Interfaces;
-using QuantConnect.Configuration;
 using System.Collections.Generic;
+using System.Globalization;
 
 namespace QuantConnect.Data.Shortable
 {
@@ -35,7 +35,7 @@ public class LocalDiskShortableProvider : IShortableProvider
 
         private string _ticker;
         private bool _scheduledCleanup;
-        private Dictionary<DateTime, long> _shortableQuantityPerDate;
+        private Dictionary<DateTime, ShortableData> _shortableDataPerDate;
 
         /// <summary>
         /// The short availability provider
@@ -51,6 +51,39 @@ public LocalDiskShortableProvider(string brokerage)
             Brokerage = brokerage.ToLowerInvariant();
         }
 
+        /// <summary>
+        /// Gets interest rate charged on borrowed shares for a given asset.
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup fee rate</param>
+        /// <param name="localTime">Time of the algorithm</param>
+        /// <returns>Fee rate. Zero if the data for the brokerage/date does not exist.</returns>
+        public decimal FeeRate(Symbol symbol, DateTime localTime)
+        {
+            if (symbol != null && GetCacheData(symbol).TryGetValue(localTime.Date, out var result))
+            {
+                return result.FeeRate;
+            }
+            // Any missing entry will be considered to be zero.
+            return 0m;
+        }
+
+        /// <summary>
+        /// Gets the Fed funds or other currency-relevant benchmark rate minus the interest rate charged on borrowed shares for a given asset.
+        /// E.g.: Interest rate - borrow fee rate = borrow rebate rate: 5.32% - 0.25% = 5.07%.
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup rebate rate</param>
+        /// <param name="localTime">Time of the algorithm</param>
+        /// <returns>Rebate fee. Zero if the data for the brokerage/date does not exist.</returns>
+        public decimal RebateRate(Symbol symbol, DateTime localTime)
+        {
+            if (symbol != null && GetCacheData(symbol).TryGetValue(localTime.Date, out var result))
+            {
+                return result.RebateFee;
+            }
+            // Any missing entry will be considered to be zero.
+            return 0m;
+        }
+
         /// <summary>
         /// Gets the quantity shortable for the Symbol at the given date.
         /// </summary>
@@ -59,22 +92,21 @@ public LocalDiskShortableProvider(string brokerage)
         /// <returns>Quantity shortable. Null if the data for the brokerage/date does not exist.</returns>
         public long? ShortableQuantity(Symbol symbol, DateTime localTime)
         {
-            var shortableQuantityPerDate = GetCacheData(symbol);
-            if (!shortableQuantityPerDate.TryGetValue(localTime.Date, out var result))
+            if (symbol != null && GetCacheData(symbol).TryGetValue(localTime.Date, out var result))
             {
-                // Any missing entry will be considered to be Shortable.
-                return null;
+                return result.ShortableQuantity;
             }
-            return result;
+            // Any missing entry will be considered to be Shortable.
+            return null;
         }
 
         /// <summary>
         /// We cache data per ticker
         /// </summary>
         /// <param name="symbol">The requested symbol</param>
-        private Dictionary<DateTime, long> GetCacheData(Symbol symbol)
+        private Dictionary<DateTime, ShortableData> GetCacheData(Symbol symbol)
         {
-            var result = _shortableQuantityPerDate;
+            var result = _shortableDataPerDate;
             if (_ticker == symbol.Value)
             {
                 return result;
@@ -89,7 +121,7 @@ public LocalDiskShortableProvider(string brokerage)
 
             // create a new collection
             _ticker = symbol.Value;
-            result = _shortableQuantityPerDate = new();
+            result = _shortableDataPerDate = new();
 
             // Implicitly trusts that Symbol.Value has been mapped and updated to the latest ticker
             var shortableSymbolFile = Path.Combine(Globals.DataFolder, symbol.SecurityType.SecurityTypeToLower(), symbol.ID.Market,
@@ -102,10 +134,15 @@ public LocalDiskShortableProvider(string brokerage)
                     // ignore empty or comment lines
                     continue;
                 }
+                // Data example. The rates, if available, are expressed in percentage.
+                // 20201221,2000,5.0700,0.2500
                 var csv = line.Split(',');
                 var date = Parse.DateTimeExact(csv[0], "yyyyMMdd");
-                var quantity = Parse.Long(csv[1]);
-                result[date] = quantity;
+                var lenght = csv.Length;
+                var shortableQuantity = csv[1].IfNotNullOrEmpty(s => long.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture));
+                var rebateRate = csv.Length > 2 ? csv[2].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)) : 0;
+                var feeRate = csv.Length > 3 ? csv[3].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)) : 0;
+                result[date] = new ShortableData(shortableQuantity, rebateRate / 100, feeRate / 100);
             }
 
             return result;
@@ -124,10 +161,12 @@ private void ClearCache()
             {
                 // create new instances so we don't need to worry about locks
                 _ticker = null;
-                _shortableQuantityPerDate = new();
+                _shortableDataPerDate = new();
 
                 ClearCache();
             });
         }
+
+        protected record ShortableData(long? ShortableQuantity, decimal RebateFee, decimal FeeRate);
     }
 }

Common/Data/Shortable/NullShortableProvider.cs

@@ -29,6 +29,29 @@ public class NullShortableProvider : IShortableProvider
         /// </summary>
         public static NullShortableProvider Instance { get; } = new ();
 
+        /// <summary>
+        /// Gets interest rate charged on borrowed shares for a given asset.
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup fee rate</param>
+        /// <param name="localTime">Time of the algorithm</param>
+        /// <returns>zero indicating that it is does have borrowing costs</returns>
+        public decimal FeeRate(Symbol symbol, DateTime localTime)
+        {
+            return 0m;       
+        }
+
+        /// <summary>
+        /// Gets the Fed funds or other currency-relevant benchmark rate minus the interest rate charged on borrowed shares for a given asset.
+        /// E.g.: Interest rate - borrow fee rate = borrow rebate rate: 5.32% - 0.25% = 5.07%.
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup rebate rate</param>
+        /// <param name="localTime">Time of the algorithm</param>
+        /// <returns>zero indicating that it is does have borrowing costs</returns>
+        public decimal RebateRate(Symbol symbol, DateTime localTime)
+        {
+            return 0m;
+        }
+
         /// <summary>
         /// Gets the quantity shortable for the Symbol at the given time.
         /// </summary>

Common/Data/Shortable/ShortableProviderPythonWrapper.cs

@@ -37,6 +37,35 @@ public ShortableProviderPythonWrapper(PyObject shortableProvider)
             _shortableProvider = shortableProvider.ValidateImplementationOf<IShortableProvider>();
         }
 
+        /// <summary>
+        /// Gets the fee rate for the Symbol at the given date.
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup fee rate</param>
+        /// <param name="localTime">Time of the algorithm</param>
+        /// <returns>zero indicating that it is does have borrowing costs</returns>
+        public decimal FeeRate(Symbol symbol, DateTime localTime)
+        {
+            using (Py.GIL())
+            {
+                return (_shortableProvider.FeeRate(symbol, localTime) as PyObject).GetAndDispose<decimal>();
+            }
+        }
+
+        /// <summary>
+        /// Gets the Fed funds or other currency-relevant benchmark rate minus the interest rate charged on borrowed shares for a given asset.
+        /// E.g.: Interest rate - borrow fee rate = borrow rebate rate: 5.32% - 0.25% = 5.07%.
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup rebate rate</param>
+        /// <param name="localTime">Time of the algorithm</param>
+        /// <returns>zero indicating that it is does have borrowing costs</returns>
+        public decimal RebateRate(Symbol symbol, DateTime localTime)
+        {
+            using (Py.GIL())
+            {
+                return (_shortableProvider.RebateRate(symbol, localTime) as PyObject).GetAndDispose<decimal>();
+            }
+        }
+
         /// <summary>
         /// Gets the quantity shortable for a <see cref="Symbol"/>, from python custom shortable provider
         /// </summary>

Common/Interfaces/IShortableProvider.cs

@@ -14,7 +14,6 @@
 */
 
 using System;
-using System.Collections.Generic;
 
 namespace QuantConnect.Interfaces
 {
@@ -23,6 +22,23 @@ namespace QuantConnect.Interfaces
     /// </summary>
     public interface IShortableProvider
     {
+        /// <summary>
+        /// Gets interest rate charged on borrowed shares for a given asset.
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup fee rate</param>
+        /// <param name="localTime">Time of the algorithm</param>
+        /// <returns>Fee rate. Zero if the data for the brokerage/date does not exist.</returns>
+        decimal FeeRate(Symbol symbol, DateTime localTime);
+
+        /// <summary>
+        /// Gets the Fed funds or other currency-relevant benchmark rate minus the interest rate charged on borrowed shares for a given asset.
+        /// Interest rate - borrow fee rate = borrow rebate rate: 5.32% - 0.25% = 5.07% 
+        /// </summary>
+        /// <param name="symbol">Symbol to lookup rebate rate</param>
+        /// <param name="localTime">Time of the algorithm</param>
+        /// <returns>Rebate fee. Zero if the data for the brokerage/date does not exist.</returns>
+        decimal RebateRate(Symbol symbol, DateTime localTime);
+
         /// <summary>
         /// Gets the quantity shortable for a <see cref="Symbol"/>.
         /// </summary>

Tests/Common/Brokerages/BrokerageModelTests.cs

@@ -572,6 +572,10 @@ public void BrokerageModelPythonWrapperWorksWithCustomShortableProvider()
 from AlgorithmImports import *
 
 class CustomShortableProvider:
+    def FeeRate(self, symbol, localTime):
+        raise ValueError(""Pepe"")
+    def RebateRate(self, symbol, localTime):
+        raise ValueError(""Pepe"")
     def ShortableQuantity(self, symbol, localTime):
         raise ValueError(""Pepe"")