Skip to content

Refactor CNBC and InsiderTrading data formats for storage efficiency #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AlexCatarino merged 2 commits into from
May 12, 2026
Merged

Refactor CNBC and InsiderTrading data formats for storage efficiency #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
19b2347
Refactor CNBC and InsiderTrading data formats for storage efficiency
AlexCatarino May 12, 2026
1e2bb09
Select dataset via config + fix path layout for Congress / WallStreet…
AlexCatarino May 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions AcquiredDisposedCode.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System.Runtime.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;

namespace QuantConnect.DataSource.QuiverQuant
{
/// <summary>
/// SEC Form 4 indicator of whether the transaction was an acquisition or a disposal
/// </summary>
[JsonConverter(typeof(StringEnumConverter))]
public enum AcquiredDisposedCode
{
/// <summary>
/// Default value used when no acquired/disposed flag is provided or the value is unrecognized
/// </summary>
[EnumMember(Value = "")]
Unknown,

/// <summary>
/// A - Share acquisition
/// </summary>
[EnumMember(Value = "A")]
Acquired,

/// <summary>
/// D - Share disposal
/// </summary>
[EnumMember(Value = "D")]
Disposed,
}
}
103 changes: 60 additions & 43 deletions DataProcessing/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,40 +26,52 @@ namespace QuantConnect.DataProcessing
/// </summary>
public class Program
{
private static readonly string VendorName = QuiverDataDownloader.VendorName;
/// <summary>
/// Entrypoint of the program
/// </summary>
/// <returns>Exit code. 0 equals successful, and any other value indicates the downloader/converter failed.</returns>
public static void Main(string[] args)
public static void Main()
{
var dataset = args[0];
var dataset = Config.Get("vendor-data-name", "cnbc").Trim().ToLowerInvariant();
var destinationDirectory = Path.Combine(
Config.Get("temp-output-directory", "/temp-output-directory"),
"alternative");
var processedDataDirectory = Path.Combine(
Config.Get("processed-data-directory", Globals.DataFolder),
"alternative");
var processingDateValue = Config.Get("processing-date", Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE"));
var processingDateValue = Config.Get("processing-date", Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE"))
?? DateTime.UtcNow.AddDays(-1).ToString("yyyyMMdd");
var processingDate = Parse.DateTimeExact(processingDateValue, "yyyyMMdd");
var processingDateLookback = Config.GetInt("processing-date-lookback", 0);
var processingStartDate = processingDate.AddDays(-processingDateLookback);

switch (dataset.ToLowerInvariant())
switch (dataset)
{
case "cnbc":
case QuiverCNBCDataDownloader.VendorDataName:
{
var processingDate = Parse.DateTimeExact(processingDateValue, "yyyyMMdd");
RunDownloader(
QuiverCNBCDataDownloader.VendorName,
QuiverCNBCDataDownloader.VendorDataName,
() => new QuiverCNBCDataDownloader(destinationDirectory, processedDataDirectory),
instance => instance.Run(processingDate));
instance =>
{
for (var date = processingStartDate; date <= processingDate; date = date.AddDays(1))
{
if (!instance.Run(date))
{
Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process " +
$"{VendorName} {QuiverCNBCDataDownloader.VendorDataName} data for date: {date:yyyy-MM-dd}");
}
}
instance.Flush();
instance.ProcessUniverse();
return true;
});
break;
}

case "governmentcontract":
case QuiverGovernmentContractDownloader.VendorDataName:
{
var processingDate = string.IsNullOrWhiteSpace(processingDateValue)
? DateTime.UtcNow.AddDays(-1)
: Parse.DateTimeExact(processingDateValue, "yyyyMMdd");

var datasetStartDate = new DateTime(2022, 4, 21);
if (processingDate < datasetStartDate)
{
Expand All @@ -68,7 +80,6 @@ public static void Main(string[] args)
}

RunDownloader(
QuiverGovernmentContractDownloader.VendorName,
QuiverGovernmentContractDownloader.VendorDataName,
() => new QuiverGovernmentContractDownloader(),
instance =>
Expand All @@ -77,68 +88,81 @@ public static void Main(string[] args)
if (!success)
{
Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process " +
$"{QuiverGovernmentContractDownloader.VendorName} {QuiverGovernmentContractDownloader.VendorDataName} data for date: {processingDate:yyyy-MM-dd}");
$"{VendorName} {QuiverGovernmentContractDownloader.VendorDataName} data for date: {processingDate:yyyy-MM-dd}");
}
instance.ProcessUniverse();
return true;
});
break;
}

case "lobbying":
case QuiverLobbyingDataDownloader.VendorDataName:
{
var processingDate = Parse.DateTimeExact(processingDateValue, "yyyyMMdd");
RunDownloader(
QuiverLobbyingDataDownloader.VendorName,
QuiverLobbyingDataDownloader.VendorDataName,
() => new QuiverLobbyingDataDownloader(destinationDirectory, processedDataDirectory),
instance => instance.Run(processingDate));
break;
}

case "congresstrading":
case QuiverCongressDataDownloader.VendorDataName:
{
var congressDestination = Path.Combine(destinationDirectory, "quiver");
RunDownloader(
QuiverCongressDataDownloader.VendorName,
QuiverCongressDataDownloader.VendorDataName,
() => new QuiverCongressDataDownloader(congressDestination),
() => new QuiverCongressDataDownloader(destinationDirectory),
instance => instance.Run());
break;
}

case "wallstreetbets":
case QuiverWallStreetBetsDataDownloader.VendorDataName:
{
var tempOutput = Config.Get("temp-output-directory", "/temp-output-directory");
RunDownloader(
QuiverWallStreetBetsDataDownloader.VendorName,
QuiverWallStreetBetsDataDownloader.VendorDataName,
() => new QuiverWallStreetBetsDataDownloader(tempOutput),
() => new QuiverWallStreetBetsDataDownloader(destinationDirectory),
instance => instance.Run());
break;
}

case "insidertrading":
case QuiverInsiderTradingDataDownloader.VendorDataName:
{
var processingStartDate = GetDateConfig("processing-start-date");
var processingEndDate = GetDateConfig("processing-end-date");
RunDownloader(
QuiverInsiderTradingDataDownloader.VendorName,
QuiverInsiderTradingDataDownloader.VendorDataName,
() => new QuiverInsiderTradingDataDownloader(destinationDirectory, processedDataDirectory),
instance => instance.Run(processingStartDate, processingEndDate));
instance =>
{
for (var date = processingStartDate; date <= processingDate; date = date.AddDays(1))
{
if (!instance.Run(date))
{
Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process " +
$"{VendorName} {QuiverInsiderTradingDataDownloader.VendorDataName} data for date: {date:yyyy-MM-dd}");
}
}
instance.Flush();
instance.ProcessUniverse();
return true;
});
break;
}

default:
Log.Error($"Unknown dataset '{dataset}'");
var validDatasets = string.Join(", ", new[]
{
QuiverCNBCDataDownloader.VendorDataName,
QuiverGovernmentContractDownloader.VendorDataName,
QuiverLobbyingDataDownloader.VendorDataName,
QuiverCongressDataDownloader.VendorDataName,
QuiverWallStreetBetsDataDownloader.VendorDataName,
QuiverInsiderTradingDataDownloader.VendorDataName,
});
Log.Error($"Unknown dataset '{dataset}'. Valid options: {validDatasets}");
break;
}

Environment.Exit(0);
}

private static void RunDownloader<T>(string vendorName, string vendorDataName, Func<T> factory, Func<T, bool> run) where T : class, IDisposable
private static void RunDownloader<T>(string vendorDataName, Func<T> factory, Func<T, bool> run) where T : class, IDisposable
{
T instance = null;
try
Expand All @@ -147,34 +171,27 @@ private static void RunDownloader<T>(string vendorName, string vendorDataName, F
}
catch (Exception err)
{
Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {vendorName} {vendorDataName} data failed to be constructed");
Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {VendorName} {vendorDataName} data failed to be constructed");
Environment.Exit(1);
}

try
{
if (!run(instance))
{
Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process {vendorName} {vendorDataName} data");
Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process {VendorName} {vendorDataName} data");
Environment.Exit(1);
}
}
catch (Exception err)
{
Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {vendorName} {vendorDataName} data exited unexpectedly");
Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {VendorName} {vendorDataName} data exited unexpectedly");
Environment.Exit(1);
}
finally
{
instance.DisposeSafely();
}
}

private static DateTime GetDateConfig(string configKey)
{
var value = Config.Get(configKey, Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE"))
?? DateTime.Today.ToString("yyyyMMdd");
return Parse.DateTimeExact(value, "yyyyMMdd");
}
}
}
Loading
Loading