Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Return null from GetHistory and Downloader #8

Merged
merged 2 commits into from
Feb 27, 2024
Merged

Return null from GetHistory and Downloader #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f86224b
feat: return null from History and Downloader methods
Romazes Feb 26, 2024
dac3508
feat: missed flags to prevent spamming for user
Romazes Feb 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions QuantConnect.IEX.Tests/IEXDataDownloaderTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ public void DownloadsHistoricalDataWithInvalidDataTestParameters(Symbol symbol,

var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType);

var downloadResponse = _downloader.Get(parameters).ToList();
var downloadResponse = _downloader.Get(parameters)?.ToList();

Assert.IsEmpty(downloadResponse);
Assert.IsNull(downloadResponse);
}

[Explicit("This tests require a iexcloud.io api key")]
Expand All @@ -54,7 +54,7 @@ public void DownloadsHistoricalDataWithValidDataTestParameters(Symbol symbol, Re

var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType);

var downloadResponse = _downloader.Get(parameters).ToList();
var downloadResponse = _downloader.Get(parameters)?.ToList();

Assert.IsNotEmpty(downloadResponse);

Expand Down
5 changes: 2 additions & 3 deletions QuantConnect.IEX.Tests/IEXDataHistoryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,7 @@ public void IEXCloudGetHistoryWithInvalidDataTestParameters(Symbol symbol, Resol
{
var slices = GetHistory(symbol, resolution, tickType, period);

Assert.IsEmpty(slices);
return;
Assert.IsNull(slices);
}

[Explicit("This tests require a iexcloud.io api key")]
Expand Down Expand Up @@ -228,7 +227,7 @@ private Slice[] GetHistory(Symbol symbol, Resolution resolution, TickType tickTy
{
var requests = new[] { CreateHistoryRequest(symbol, resolution, tickType, period) };

var slices = iexDataProvider.GetHistory(requests, TimeZones.Utc).ToArray();
var slices = iexDataProvider.GetHistory(requests, TimeZones.Utc)?.ToArray();
Log.Trace("Data points retrieved: " + iexDataProvider.DataPointCount);
Log.Trace("tick Type: " + tickType);
return slices;
Expand Down
20 changes: 10 additions & 10 deletions QuantConnect.IEX/IEXDataDownloader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public void Dispose()
/// </summary>
/// <param name="dataDownloaderGetParameters">model class for passing in parameters for historical data</param>
/// <returns>Enumerable of base data for this symbol</returns>
public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
public IEnumerable<BaseData>? Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
Expand All @@ -52,19 +52,21 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa

if (tickType != TickType.Trade)
{
yield break;
Log.Error($"{nameof(IEXDataDownloader)}.{nameof(Get)}: Not supported data type - {tickType}. " +
"Currently available support only for historical of type - TradeBar");
return null;
}

if (resolution != Resolution.Daily && resolution != Resolution.Minute)
if (resolution != Resolution.Daily && resolution != Resolution.Minute)
{
Log.Error($"{nameof(IEXDataDownloader)}.{nameof(Get)}: InvalidResolution. {resolution} resolution not supported, no history returned");
yield break;
return null;
}

if (endUtc < startUtc)
{
Log.Error($"{nameof(IEXDataDownloader)}.{nameof(Get)}:InvalidDateRange. The history request start date must precede the end date, no history returned");
yield break;
return null;
}

var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
Expand All @@ -75,7 +77,7 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
endUtc,
typeof(TradeBar),
symbol,
resolution,
resolution,
exchangeHours,
dataTimeZone,
resolution,
Expand All @@ -85,10 +87,8 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
TickType.Trade)
};

foreach (var slice in _handler.GetHistory(historyRequests, TimeZones.EasternStandard))
{
yield return slice[symbol];
}

return _handler.GetHistory(historyRequests, TimeZones.EasternStandard)?.Select(slice => (BaseData)slice[symbol]);
}
}
}
83 changes: 65 additions & 18 deletions QuantConnect.IEX/IEXDataProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,21 @@ public class IEXDataProvider : SynchronizingHistoryProvider, IDataQueueHandler
/// </summary>
private static bool _invalidHistoryDataTypeWarningFired;

/// <summary>
/// Indicates whether the warning for invalid <see cref="SecurityType"/> has been fired.
/// </summary>
private bool _invalidSecurityTypeWarningFired;

/// <summary>
/// Indicates whether a warning for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC.
/// </summary>
private bool _invalidStartTimeWarningFired;

/// <summary>
/// Indicates whether a warning for an invalid <see cref="Resolution"/> has been fired, where the resolution is neither daily nor minute-based.
/// </summary>
private bool _invalidResolutionWarningFired;

/// <summary>
/// Represents two clients: one for the trade channel and another for the top-of-book channel.
/// </summary>
Expand Down Expand Up @@ -491,7 +506,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
/// <param name="requests">The historical data requests</param>
/// <param name="sliceTimeZone">The time zone used when time stamping the slice instances</param>
/// <returns>An enumerable of the slices of data covering the span specified in each request</returns>
public override IEnumerable<Slice> GetHistory(IEnumerable<Data.HistoryRequest> requests, DateTimeZone sliceTimeZone)
public override IEnumerable<Slice>? GetHistory(IEnumerable<Data.HistoryRequest> requests, DateTimeZone sliceTimeZone)
{
// Create subscription objects from the configs
var subscriptions = new List<Subscription>();
Expand All @@ -506,44 +521,61 @@ public override IEnumerable<Slice> GetHistory(IEnumerable<Data.HistoryRequest> r
"Currently available support only for historical of type - TradeBar");
_invalidHistoryDataTypeWarningFired = true;
}
return Enumerable.Empty<Slice>();
continue;
}

if (request.Symbol.SecurityType != SecurityType.Equity)
if (!CanSubscribe(request.Symbol))
{
Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'");
return Enumerable.Empty<Slice>();
if (!_invalidSecurityTypeWarningFired)
{
Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'");
_invalidSecurityTypeWarningFired = true;
}
continue;
}

if (request.StartTimeUtc >= request.EndTimeUtc)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned.");
return Enumerable.Empty<Slice>();
if (!_invalidStartTimeWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned.");
_invalidStartTimeWarningFired = true;
}
continue;
}

if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
{
if (!_invalidResolutionWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution.");
_invalidResolutionWarningFired = true;
}
continue;
}

var history = ProcessHistoryRequests(request);
var subscription = CreateSubscription(request, history);
subscriptions.Add(subscription);
}

if (subscriptions.Count == 0)
{
return null;
}

return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone);
}

/// <summary>
/// Populate request data
/// </summary>
private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request)
private IEnumerable<BaseData>? ProcessHistoryRequests(Data.HistoryRequest request)
{
var ticker = request.Symbol.ID.Symbol;
var start = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc);
var end = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc);

if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
{
Log.Error("IEXDataProvider.GetHistory(): History calls for IEX only support daily & minute resolution.");
yield break;
}

Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{start} - {end}].");

var span = end - start;
Expand Down Expand Up @@ -602,6 +634,21 @@ private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request
}
}

return GetHistoryRequestByUrls(urls, start, end, request.Resolution, request.DataNormalizationMode, request.Symbol);
}

/// <summary>
/// Retrieves historical data by making requests to the specified URLs.
/// </summary>
/// <param name="urls">The list of URLs to retrieve historical data from.</param>
/// <param name="startDateTime">The start date and time for the historical data. (use to skip data which not passed condition)</param>
/// <param name="endDateTime">The end date and time for the historical data. (use to skip data which not passed condition)</param>
/// <param name="resolution">The <see cref="Resolution"/> of the historical data.</param>
/// <param name="dataNormalizationMode">The <seealso cref="DataNormalizationMode"/> the historical data.</param>
/// <param name="symbol">The <seealso cref="Symbol"/> for which historical data is being retrieved.</param>
/// <returns>An enumerable collection of <see cref="BaseData"/> containing the retrieved historical data.</returns>
private IEnumerable<BaseData> GetHistoryRequestByUrls(List<string> urls, DateTime startDateTime, DateTime endDateTime, Resolution resolution, DataNormalizationMode dataNormalizationMode, Symbol symbol)
{
foreach (var url in urls)
{
var response = ExecuteGetRequest(url);
Expand Down Expand Up @@ -631,7 +678,7 @@ private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request
period = TimeSpan.FromDays(1);
}

if (date < start || date > end)
if (date < startDateTime || date > endDateTime)
{
continue;
}
Expand All @@ -645,8 +692,8 @@ private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request

decimal open, high, low, close, volume;

if (request.Resolution == Resolution.Daily &&
request.DataNormalizationMode == DataNormalizationMode.Raw)
if (resolution == Resolution.Daily &&
dataNormalizationMode == DataNormalizationMode.Raw)
{
open = item["uOpen"].Value<decimal>();
high = item["uHigh"].Value<decimal>();
Expand All @@ -663,7 +710,7 @@ private IEnumerable<BaseData> ProcessHistoryRequests(Data.HistoryRequest request
volume = item["volume"].Value<int>();
}

var tradeBar = new TradeBar(date, request.Symbol, open, high, low, close, volume, period);
var tradeBar = new TradeBar(date, symbol, open, high, low, close, volume, period);

yield return tradeBar;
}
Expand Down
Loading