Generic: DataProvider (#25)

* feat: implement DataProvider + test

* feat: delisted\mapping test cases

* rename: namespaces to new format

* revert: rename of dataType and demonstration

* rename: missed namespace word by template from another ones

* feat: IDataDownloader && IDataQueueHandler
feat: tests

DemonstrationUniverse.cs

@@ -14,9 +14,7 @@
  *
 */
 
-using System;
 using System.Linq;
-using QuantConnect.Data;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.DataSource;
 

MyCustomDataDownloader.cs

@@ -0,0 +1,59 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using System.Collections.Generic;
+using QuantConnect.Util;
+
+namespace QuantConnect.Lean.DataSource.MyCustom
+{
+    /// <summary>
+    /// Data downloader class for pulling data from Data Provider
+    /// </summary>
+    public class MyCustomDataDownloader : IDataDownloader, IDisposable
+    {
+        /// <inheritdoc cref="MyCustomDataProvider"/>
+        private readonly MyCustomDataProvider _myCustomDataProvider;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="MyCustomDataDownloader"/>
+        /// </summary>
+        public MyCustomDataDownloader()
+        {
+            _myCustomDataProvider = new MyCustomDataProvider();
+        }
+
+        /// <summary>
+        /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
+        /// </summary>
+        /// <param name="dataDownloaderGetParameters">Parameters for the historical data request</param>
+        /// <returns>Enumerable of base data for this symbol</returns>
+        /// <exception cref="NotImplementedException"></exception>
+        public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetParameters)
+        {
+            throw new NotImplementedException();
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            _myCustomDataProvider?.DisposeSafely();
+        }
+    }
+}

MyCustomDataProvider.cs

@@ -0,0 +1,161 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.HistoricalData;
+
+namespace QuantConnect.Lean.DataSource.MyCustom
+{
+    /// <summary>
+    /// Implementation of Custom Data Provider
+    /// </summary>
+    public class MyCustomDataProvider : SynchronizingHistoryProvider, IDataQueueHandler
+    {
+        /// <summary>
+        /// <inheritdoc cref="IDataAggregator"/>
+        /// </summary>
+        private readonly IDataAggregator _dataAggregator;
+
+        /// <summary>
+        /// <inheritdoc cref="EventBasedDataQueueHandlerSubscriptionManager"/>
+        /// </summary>
+        private readonly EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager;
+
+        /// <summary>
+        /// Returns true if we're currently connected to the Data Provider
+        /// </summary>
+        public bool IsConnected { get; }
+
+        /// <inheritdoc cref="HistoryProviderBase.Initialize(HistoryProviderInitializeParameters)"/>
+        public override void Initialize(HistoryProviderInitializeParameters parameters)
+        { }
+
+        /// <inheritdoc cref="HistoryProviderBase.GetHistory(IEnumerable{HistoryRequest}, DateTimeZone)"/>
+        public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
+        {
+            // Create subscription objects from the configs
+            var subscriptions = new List<Subscription>();
+            foreach (var request in requests)
+            {
+                // Retrieve the history for the current request
+                var history = GetHistory(request);
+
+                if (history == null)
+                {
+                    // If history is null, it indicates that the request contains wrong parameters
+                    // Handle the case where the request parameters are incorrect
+                    continue;
+                }
+
+                var subscription = CreateSubscription(request, history);
+                subscriptions.Add(subscription);
+            }
+
+            // Validate that at least one subscription is valid; otherwise, return null
+            if (subscriptions.Count == 0)
+            {
+                return null;
+            }
+
+            return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone);
+        }
+
+        /// <summary>
+        /// Subscribe to the specified configuration
+        /// </summary>
+        /// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
+        /// <param name="newDataAvailableHandler">handler to be fired on new data available</param>
+        /// <returns>The new enumerator for this subscription request</returns>
+        public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
+        {
+            if (!CanSubscribe(dataConfig.Symbol))
+            {
+                return null;
+            }
+
+            var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler);
+            _subscriptionManager.Subscribe(dataConfig);
+
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Removes the specified configuration
+        /// </summary>
+        /// <param name="dataConfig">Subscription config to be removed</param>
+        public void Unsubscribe(SubscriptionDataConfig dataConfig)
+        {
+            _subscriptionManager.Unsubscribe(dataConfig);
+            _dataAggregator.Remove(dataConfig);
+        }
+
+        /// <summary>
+        /// Sets the job we're subscribing for
+        /// </summary>
+        /// <param name="job">Job we're subscribing for</param>
+        /// <exception cref="NotImplementedException"></exception>
+        public void SetJob(Packets.LiveNodePacket job)
+        {
+            throw new NotImplementedException();
+        }
+
+        /// <summary>
+        /// Dispose of unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            _dataAggregator?.DisposeSafely();
+            _subscriptionManager?.DisposeSafely();
+            throw new NotImplementedException();
+        }
+
+        /// <summary>
+        /// Gets the history for the requested security
+        /// </summary>
+        /// <param name="request">The historical data request</param>
+        /// <returns>An enumerable of BaseData points</returns>
+        private IEnumerable<BaseData> GetHistory(HistoryRequest request)
+        {
+            if (!CanSubscribe(request.Symbol))
+            {
+                return null;
+            }
+
+            throw new NotImplementedException();
+        }
+
+        /// <summary>
+        /// Checks if this Data provider supports the specified symbol
+        /// </summary>
+        /// <param name="symbol">The symbol</param>
+        /// <returns>returns true if Data Provider supports the specified symbol; otherwise false</returns>
+        private bool CanSubscribe(Symbol symbol)
+        {
+            if (symbol.Value.IndexOfInvariant("universe", true) != -1 || symbol.IsCanonical())
+            {
+                return false;
+            }
+
+            throw new NotImplementedException();
+        }
+    }
+}

QuantConnect.DataSource.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
     <RootNamespace>QuantConnect.DataSource</RootNamespace>
@@ -9,6 +9,7 @@
 
   <ItemGroup>
     <PackageReference Include="QuantConnect.Common" Version="2.5.*" />
+	  <PackageReference Include="QuantConnect.Lean.Engine" Version="2.5.*" />
     <PackageReference Include="protobuf-net" Version="3.1.33" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
   </ItemGroup>

tests/MyCustomDataDownloaderTests.cs

@@ -0,0 +1,41 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using NUnit.Framework;
+using System.Collections.Generic;
+using QuantConnect.Lean.DataSource.MyCustom;
+
+namespace QuantConnect.DataLibrary.Tests
+{
+    [TestFixture]
+    public class MyCustomDataDownloaderTests
+    {
+        private static IEnumerable<TestCaseData> DownloadTestParameters => MyCustomDataProviderHistoryTests.TestParameters;
+
+        [TestCaseSource(nameof(DownloadTestParameters))]
+        public void DownloadHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool isThrowNotImplementedException)
+        {
+            var myCustomDownloader = new MyCustomDataDownloader();
+
+            var request = MyCustomDataProviderHistoryTests.GetHistoryRequest(resolution, tickType, symbol, period);
+
+            var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType);
+
+            Assert.Throws<NotImplementedException>(() => myCustomDownloader.Get(parameters));
+        }
+    }
+}

tests/MyCustomDataProviderHistoryTests.cs

@@ -0,0 +1,109 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Tests;
+using QuantConnect.Lean.DataSource.MyCustom;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Tests.Common.Exceptions;
+
+namespace QuantConnect.DataLibrary.Tests
+{
+    [TestFixture]
+    public class MyCustomDataProviderHistoryTests
+    {
+        /// <inheritdoc cref="MyCustomDataProvider"/>
+        private readonly MyCustomDataProvider _historyDataProvider = new();
+
+        internal static IEnumerable<TestCaseData> TestParameters
+        {
+            get
+            {
+                TestGlobals.Initialize();
+                var equity = Symbol.Create("SPY", SecurityType.Equity, Market.USA);
+                var option = Symbol.Create("SPY", SecurityType.Option, Market.USA);
+
+                yield return new TestCaseData(equity, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), true)
+                    .SetDescription("Valid parameters - Daily resolution, 15 days period.")
+                    .SetCategory("Valid");
+
+                yield return new TestCaseData(equity, Resolution.Hour, TickType.Quote, TimeSpan.FromDays(2), true)
+                    .SetDescription("Valid parameters - Hour resolution, 2 days period.")
+                    .SetCategory("Valid");
+
+                yield return new TestCaseData(option, Resolution.Second, TickType.Trade, TimeSpan.FromMinutes(60), false)
+                    .SetDescription("Invalid Symbol - Canonical doesn't support")
+                    .SetCategory("Invalid");
+
+                /// <see cref="Slice.Delistings"/>
+                yield return new TestCaseData(Symbol.Create("AAA.1", SecurityType.Equity, Market.USA), Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), true)
+                    .SetDescription("Delisted Symbol - the DataSource supports the history of delisted ones or not")
+                    .SetCategory("Valid/Invalid");
+
+                /// <see cref="Slice.SymbolChangedEvents"/>
+                yield return new TestCaseData(Symbol.Create("SPWR", SecurityType.Equity, Market.USA), Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), true)
+                    .SetDescription("Mapping Symbol")
+                    .SetCategory("Valid");
+            }
+        }
+
+        [Test, TestCaseSource(nameof(TestParameters))]
+        public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool isThrowNotImplementedException)
+        {
+            var request = GetHistoryRequest(resolution, tickType, symbol, period);
+
+            try
+            {
+                IEnumerable<Slice> slices = _historyDataProvider.GetHistory(new[] { request }, TimeZones.Utc)?.ToList();
+                Assert.IsNull(slices);
+            }
+            catch (NotImplementedException)
+            {
+                Assert.IsTrue(isThrowNotImplementedException);
+            }
+        }
+
+        internal static HistoryRequest GetHistoryRequest(Resolution resolution, TickType tickType, Symbol symbol, TimeSpan period)
+        {
+            var utcNow = DateTime.UtcNow;
+            var dataType = LeanData.GetDataType(resolution, tickType);
+            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
+
+            var exchangeHours = marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
+            var dataTimeZone = marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
+
+            return new HistoryRequest(
+                startTimeUtc: utcNow.Add(-period),
+                endTimeUtc: utcNow,
+                dataType: dataType,
+                symbol: symbol,
+                resolution: resolution,
+                exchangeHours: exchangeHours,
+                dataTimeZone: dataTimeZone,
+                fillForwardResolution: resolution,
+                includeExtendedMarketHours: true,
+                isCustomData: false,
+                DataNormalizationMode.Raw,
+                tickType: tickType
+                );
+        }
+    }
+}