Permalink
Switch branches/tags
algo-timeout-stacktrace alpaca-brokerage alpha-value-monthly bitfinex bug-337-add-support-for-python-project-directories bug-2187-remove-framework-pop-avg-score-warmup-period bug-2189-parralelize-fine-fundamental-reading bug-2288-pythonnet-memory-leak bug-2357-fix-insight-close-time bug-2381-daily-benchmark-for-backtesting bug-2381-setbenchmark-fix bug-2504-ib-always-restore-data-subscriptions bug-2513-performcashsync-once bug-2532-apply-splits-in-live-mode bug-2541-live-mode-always-apply-splits-never-apply-dividends bug-2569-apply-dividends-in-live-paper bug-2569-double-dividend-application bug-2611-live-trading-sync-algorithm-status-update bug-2762-market-order-fills-stale-prices bugfix-insight-close-time crypto-symbol-length debugging-api-logging desktop-mk-ii docker-file-lean-foundation-updates feature-452-net-core feature-1040-object-store feature-1093-vwap-order-type feature-1418-buying-power-order-fee-contexts feature-1418-fee-model-units feature-2003-kraken-exchange feature-2047-split-dividend-api feature-2060-multi-leg-currency-conversion feature-2068-refactor-regression-test-suite feature-2271-IRegressionAlgorithmDefinition-CanRunLocally feature-2378-fix-stream-reader-disposal feature-2378-generator-factors feature-2378-live-factor-files feature-2378-minor-split-dividend-fixes feature-2378-split-dividend-improvements feature-2581-multiple-risk-models feature-timestamped-packets feature/1418-fee-model-context feature/2606-custom-brokerage-message-handler features-1998-2219-portfolio-implementation-v2 fix-python-algorithm-loading fix-vix-futures-scale-factor fsdf-thread-count-logging ib-restart-handler-fix ibrokeragemodel-isshortsellingallowed live-test-move-addsubscription-removesubscription log-splits-dividends master multiple-brokerages quandl-live-extra-logging refactor-1418-buying-power-model-context refactor-2491-livetradingdatafeed-will-use-subscriptionsynchronizer refactor-2567-fill-fee-model-invocation refactor-remove-job-packet release-engine-test smarter-live-chart-subscriptions symbol-tostring-no-subscription test-insight-scoring tick_not_decimal track-all-security-subscriptions trade-crypto-history-requests tweak-2530-log-split-dividend-prices tweak-configure-await-extension tweak-improve-map-file-read-time tweak-make-InsightFromSerializedInsight-public tweak-minor-changes-splits-dividends
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
347 lines (301 sloc) 14.1 KB
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using QuantConnect.Data;
using QuantConnect.Data.Auxiliary;
using QuantConnect.Data.Market;
namespace QuantConnect.ToolBox
{
/// <summary>
/// Generates a factor file from a list of splits and dividends for a specified equity
/// </summary>
public class FactorFileGenerator
{
/// <summary>
/// The symbol for which the factor file is being generated
/// </summary>
public Symbol Symbol { get; set; }
/// <summary>
/// Data for this equity at daily resolution
/// </summary>
private readonly List<TradeBar> _dailyDataForEquity;
/// <summary>
/// The last date in the _dailyEquityData
/// </summary>
private readonly DateTime _lastDateFromEquityData;
/// <summary>
/// Constructor for the FactorFileGenerator
/// </summary>
/// <param name="symbol">The equity for which the factor file respresents</param>
/// <param name="pathForDailyEquityData">The path to the daily data for the specified equity</param>
public FactorFileGenerator(Symbol symbol, string pathForDailyEquityData)
{
Symbol = symbol;
_dailyDataForEquity = ReadDailyEquityData(pathForDailyEquityData);
_lastDateFromEquityData = _dailyDataForEquity.Last().Time;
}
/// <summary>
/// Create FactorFile instance
/// </summary>
/// <param name="dividendSplitList">List of Dividends and Splits</param>
/// <returns><see cref="FactorFile"/> instance</returns>
public FactorFile CreateFactorFile(List<BaseData> dividendSplitList)
{
var orderedDividendSplitQueue = new Queue<BaseData>(
CombineIntraDayDividendSplits(dividendSplitList)
.OrderByDescending(x => x.Time));
var factorFileRows = new List<FactorFileRow>
{
// First Factor Row is set far into the future and by definition has 1 for both price and split factors
new FactorFileRow(
Time.EndOfTime,
priceFactor: 1,
splitFactor: 1
)
};
return RecursivlyGenerateFactorFile(orderedDividendSplitQueue, factorFileRows);
}
/// <summary>
/// If dividend and split occur on the same day,
/// combine them into IntraDayDividendSplit object
/// </summary>
/// <param name="splitDividendList">List of split and dividends</param>
/// <returns>A list of splits, dividends with intraday split and dividends combined into <see cref="IntraDayDividendSplit"/></returns>
private List<BaseData> CombineIntraDayDividendSplits(List<BaseData> splitDividendList)
{
var splitDividendCollection = new Collection<BaseData>(splitDividendList);
var dateKeysLookup = splitDividendCollection.GroupBy(x => x.Time)
.OrderByDescending(x => x.Key)
.Select(group => group)
.ToList();
var baseDataList = new List<BaseData>();
foreach (var kvpLookup in dateKeysLookup)
{
if (kvpLookup.Count() > 1)
{
// Intraday dividend split found
var dividend = kvpLookup.First(x => x.GetType() == typeof(Dividend)) as Dividend;
var split = kvpLookup.First(x => x.GetType() == typeof(Split)) as Split;
baseDataList.Add(new IntraDayDividendSplit(split, dividend));
}
else
{
baseDataList.Add(kvpLookup.First());
}
}
return baseDataList;
}
/// <summary>
/// Recursively generate a <see cref="FactorFile"/>
/// </summary>
/// <param name="orderedDividendSplits">Queue of dividends and splits ordered by date</param>
/// <param name="factorFileRows">The list of factor file rows</param>
/// <returns><see cref="FactorFile"/> instance</returns>
private FactorFile RecursivlyGenerateFactorFile(Queue<BaseData> orderedDividendSplits, List<FactorFileRow> factorFileRows)
{
// If there is no more dividends or splits, return
if (!orderedDividendSplits.Any())
{
factorFileRows.Add(CreateLastFactorFileRow(factorFileRows));
return new FactorFile(Symbol.ID.Symbol, factorFileRows);
}
var nextEvent = orderedDividendSplits.Dequeue();
// If there is no more daily equity data to use, return
if (_lastDateFromEquityData > nextEvent.Time)
{
factorFileRows.Add(CreateLastFactorFileRow(factorFileRows));
return new FactorFile(Symbol.ID.Symbol, factorFileRows);
}
var nextFactorFileRow = CalculateNextFactorFileRow(factorFileRows, nextEvent);
if (nextFactorFileRow != null)
factorFileRows.Add(nextFactorFileRow);
return RecursivlyGenerateFactorFile(orderedDividendSplits, factorFileRows);
}
/// <summary>
/// Create the last FileFactorRow.
/// Represents the earliest date that the daily equity data contains.
/// </summary>
/// <param name="factorFileRows">The list of factor file rows</param>
/// <returns><see cref="FactorFileRow"/></returns>
private FactorFileRow CreateLastFactorFileRow(List<FactorFileRow> factorFileRows)
{
return new FactorFileRow(
_dailyDataForEquity.Last().Time.Date,
factorFileRows.Last().PriceFactor,
factorFileRows.Last().SplitFactor
);
}
/// <summary>
/// Calculates the next <see cref="FactorFileRow"/>
/// </summary>
/// <param name="factorFileRows">The current list of factorFileRows</param>
/// <param name="nextEvent">The next dividend, split or intradayDividendSplit</param>
/// <returns>A single factor file row</returns>
private FactorFileRow CalculateNextFactorFileRow(List<FactorFileRow> factorFileRows, BaseData nextEvent)
{
FactorFileRow nextFactorFileRow;
var t = nextEvent.GetType();
switch (t.Name)
{
case "Dividend":
nextFactorFileRow = CalculateNextDividendFactor(nextEvent, factorFileRows.Last());
break;
case "Split":
nextFactorFileRow = CalculateNextSplitFactor(nextEvent, factorFileRows.Last());
break;
case "IntraDayDividendSplit":
nextFactorFileRow = CalculateIntradayDividendSplit((IntraDayDividendSplit)nextEvent, factorFileRows.Last());
break;
default:
throw new ArgumentException("Unhandled BaseData type for FactorFileGenerator.");
}
return nextFactorFileRow;
}
/// <summary>
/// Generates the <see cref="FactorFileRow"/> that represents a intraday dividend split.
/// Applies the dividend first.
/// </summary>
/// <param name="intraDayDividendSplit"><see cref="IntraDayDividendSplit"/> instance that holds the intraday dividend and split information</param>
/// <param name="last">The last <see cref="FactorFileRow"/> generated recursivly</param>
/// <returns><see cref="FactorFileRow"/> that represents an intraday dividend and split</returns>
private FactorFileRow CalculateIntradayDividendSplit(IntraDayDividendSplit intraDayDividendSplit, FactorFileRow last)
{
var row = CalculateNextDividendFactor(intraDayDividendSplit.Dividend, last);
return CalculateNextSplitFactor(intraDayDividendSplit.Split, row);
}
/// <summary>
/// Calculates the price factor of a <see cref="Dividend"/>
/// </summary>
/// <param name="dividend">The next dividend</param>
/// <param name="previousFactorFileRow">The previous <see cref="FactorFileRow"/> generated</param>
/// <returns><see cref="FactorFileRow"/> that represents the dividend event</returns>
private FactorFileRow CalculateNextDividendFactor(BaseData dividend, FactorFileRow previousFactorFileRow)
{
var eventDayData = GetDailyDataForDate(dividend.Time);
// If you don't have the equity data nothing can be calculated
if (eventDayData == null)
{
return null;
}
TradeBar previousClosingPrice = FindPreviousTradableDayClosingPrice(eventDayData.Time);
var priceFactor = previousFactorFileRow.PriceFactor - (dividend.Value / ((previousClosingPrice.Close) * previousFactorFileRow.SplitFactor));
return new FactorFileRow(
previousClosingPrice.Time,
priceFactor.RoundToSignificantDigits(7),
previousFactorFileRow.SplitFactor,
previousClosingPrice.Close
);
}
/// <summary>
/// Calculates the split factor of a <see cref="Split"/>
/// </summary>
/// <param name="split">The next <see cref="Split"/></param>
/// <param name="previousFactorFileRow">The previous <see cref="FactorFileRow"/> generated</param>
/// <returns><see cref="FactorFileRow"/> that represents the split event</returns>
private FactorFileRow CalculateNextSplitFactor(BaseData split, FactorFileRow previousFactorFileRow)
{
var eventDayData = GetDailyDataForDate(split.Time);
// If you don't have the equity data nothing can be done
if (eventDayData == null)
{
return null;
}
TradeBar previousClosingPrice = FindPreviousTradableDayClosingPrice(eventDayData.Time);
return new FactorFileRow(
previousClosingPrice.Time,
previousFactorFileRow.PriceFactor,
(previousFactorFileRow.SplitFactor * split.Value).RoundToSignificantDigits(6),
previousClosingPrice.Close
);
}
/// <summary>
/// Gets the data for a specified date
/// </summary>
/// <param name="date">The current specified date</param>
/// <returns><see cref="TradeBar"/>representing that date</returns>
private TradeBar GetDailyDataForDate(DateTime date)
{
return _dailyDataForEquity.FirstOrDefault(x => x.Time.Date == date.Date);
}
/// <summary>
/// Gets the data for the previous tradable day
/// </summary>
/// <param name="date">The current specified date</param>
/// <returns>The last tradeble days data</returns>
private TradeBar FindPreviousTradableDayClosingPrice(DateTime date)
{
TradeBar previousDayData = null;
var lastDateforData = _dailyDataForEquity.Last();
while (previousDayData == null && date > lastDateforData.EndTime)
{
previousDayData = _dailyDataForEquity.FirstOrDefault(x => x.Time == date.AddDays(-1));
date = date.AddDays(-1);
}
return previousDayData;
}
/// <summary>
/// Read the daily equity date from file
/// </summary>
/// <param name="pathForDailyEquityData">Path the the daily data</param>
/// <returns>A list of <see cref="TradeBar"/> read from file</returns>
private List<TradeBar> ReadDailyEquityData(string pathForDailyEquityData)
{
using (var zipToOpen = new FileStream(pathForDailyEquityData, FileMode.Open))
{
using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Read))
{
foreach (var entry in archive.Entries)
{
var parser = new LeanParser();
var stream = entry.Open();
return parser.Parse(pathForDailyEquityData, stream)
.OrderByDescending(x => x.Time)
.Select(x => (TradeBar)x)
.ToList();
}
}
}
return new List<TradeBar>();
}
/// <summary>
/// Pairs split and dividend data into one type
/// </summary>
private class IntraDayDividendSplit : BaseData
{
public Split Split { get; }
public Dividend Dividend { get; }
public IntraDayDividendSplit(Split split, Dividend dividend)
{
if (split == null)
{
throw new ArgumentNullException("split");
}
if (dividend == null)
{
throw new ArgumentNullException("dividend");
}
Split = split;
Dividend = dividend;
Time = Split.Time;
}
}
}
}