Standarize universe data type implementation

DemonstrationUniverse.cs

@@ -38,18 +38,32 @@ public override void Initialize()
             SetCash(100000);
 
             // add a custom universe data source (defaults to usa-equity)
-            AddUniverse<MyCustomDataUniverseType>("MyCustomDataUniverseType", Resolution.Daily, data =>
+            var universe = AddUniverse<MyCustomDataUniverse>(data =>
             {
-                foreach (var datum in data)
+                foreach (MyCustomDataUniverse datum in data)
                 {
                     Log($"{datum.Symbol},{datum.SomeCustomProperty},{datum.SomeNumericProperty}");
                 }
 
                 // define our selection criteria
-                return from d in data
+                return from MyCustomDataUniverse d in data
                        where d.SomeCustomProperty == "buy"
                        select d.Symbol;
             });
+
+            var history = History(universe, 1).ToList();
+            if (history.Count != 1)
+            {
+                throw new System.Exception($"Unexpected historical data count!");
+            }
+            foreach (var dataForDate in history)
+            {
+                var coarseData = dataForDate.ToList();
+                if (coarseData.Count < 300)
+                {
+                    throw new System.Exception($"Unexpected historical universe data!");
+                }
+            }
         }
 
         /// <summary>

DemonstrationUniverse.py

@@ -16,7 +16,7 @@
 ### <summary>
 ### Example algorithm using the custom data type as a source of alpha
 ### </summary>
-class CustomDataUniverse(QCAlgorithm): 
+class CustomDataUniverse(QCAlgorithm):
     def Initialize(self):
         ''' Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. '''
 
@@ -28,7 +28,15 @@ def Initialize(self):
         self.SetCash(100000)
 
         # add a custom universe data source (defaults to usa-equity)
-        self.AddUniverse(MyCustomDataUniverseType, "MyCustomDataUniverseType", Resolution.Daily, self.UniverseSelection)
+        universe = self.AddUniverse(MyCustomDataUniverse, self.UniverseSelection)
+
+        history = self.History(universe, TimeSpan(1, 0, 0, 0))
+        if len(history) != 1:
+            raise ValueError(f"Unexpected history count {len(history)}! Expected 1")
+
+        for dataForDate in history:
+            if len(dataForDate) < 300:
+                raise ValueError(f"Unexpected historical universe data!")
 
     def UniverseSelection(self, data):
         ''' Selected the securities
@@ -37,7 +45,7 @@ def UniverseSelection(self, data):
         :return: List of Symbol objects '''
 
         for datum in data:
-            self.Log(f"{datum.Symbol},{datum.Followers},{datum.DayPercentChange},{datum.WeekPercentChange}")
+            self.Log(f"{datum.Symbol},{datum.SomeNumericProperty},{datum.SomeCustomProperty}")
 
         # define our selection criteria
         return [d.Symbol for d in data if d.SomeCustomProperty == 'buy']

MyCustomDataUniverseType.cs → MyCustomDataUniverse.cs

@@ -21,34 +21,35 @@
 using QuantConnect.Data;
 using System.Collections.Generic;
 using System.Globalization;
+using QuantConnect.Data.UniverseSelection;
 
 namespace QuantConnect.DataSource
 {
     /// <summary>
     /// Example custom data type
     /// </summary>
     [ProtoContract(SkipConstructor = true)]
-    public class MyCustomDataUniverseType : BaseData
+    public class MyCustomDataUniverse : BaseDataCollection
     {
         /// <summary>
-        /// Some custom data property
+        /// Time passed between the date of the data and the time the data became available to us
         /// </summary>
-        public string SomeCustomProperty { get; set; } 
+        private readonly static TimeSpan _period = TimeSpan.FromDays(1);
 
         /// <summary>
         /// Some custom data property
         /// </summary>
-        public decimal SomeNumericProperty { get; set; }
+        public string SomeCustomProperty { get; set; }
 
         /// <summary>
-        /// Time passed between the date of the data and the time the data became available to us
+        /// Some custom data property
         /// </summary>
-        public TimeSpan Period { get; set; } = TimeSpan.FromDays(1);
+        public decimal SomeNumericProperty { get; set; }
 
         /// <summary>
         /// Time the data became available
         /// </summary>
-        public override DateTime EndTime => Time + Period;
+        public override DateTime EndTime => Time + _period;
 
         /// <summary>
         /// Return the URL string source of the file. This will be converted to a stream
@@ -67,7 +68,8 @@ public override SubscriptionDataSource GetSource(SubscriptionDataConfig config,
                     "universe",
                     $"{date.ToStringInvariant(DateFormat.EightCharacter)}.csv"
                 ),
-                SubscriptionTransportMedium.LocalFile
+                SubscriptionTransportMedium.LocalFile,
+                FileFormat.FoldingCollection
             );
         }
 
@@ -85,12 +87,12 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date
 
             var someNumericProperty = decimal.Parse(csv[2], NumberStyles.Any, CultureInfo.InvariantCulture); 
 
-            return new MyCustomDataUniverseType
+            return new MyCustomDataUniverse
             {
                 Symbol = new Symbol(SecurityIdentifier.Parse(csv[0]), csv[1]),
                 SomeNumericProperty = someNumericProperty,
                 SomeCustomProperty = csv[3],
-                Time =  date - Period,
+                Time =  date - _period,
                 Value = someNumericProperty
             };
         }
@@ -137,5 +139,22 @@ public override DateTimeZone DataTimeZone()
         {
             return DateTimeZone.Utc;
         }
+
+        /// <summary>
+        /// Clones this instance
+        /// </summary>
+        public override BaseData Clone()
+        {
+            return new MyCustomDataUniverse
+            {
+                Symbol = Symbol,
+                Time = Time,
+                Data = Data,
+                Value = Value,
+
+                SomeNumericProperty = SomeNumericProperty,
+                SomeCustomProperty = SomeCustomProperty,
+            };
+        }
     }
 }