Skip to content

Refactor optimization statistics serialization #8984

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Refactor optimization statistics serialization #8984

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
50d3cbf
Refactor optimization result stats serialization
jhonabreul Sep 19, 2025
054f795
Handle custom optimization statistics serialization
jhonabreul Sep 22, 2025
6dd1459
Support custom statistics
jhonabreul Sep 22, 2025
bc6ae5b
Support newest Lean statistics
jhonabreul Sep 22, 2025
a734414
Add more tests
jhonabreul Sep 23, 2025
2d88a06
Make indices reserved statistic names
jhonabreul Sep 23, 2025
f212b40
Minor fixes and cleanup
jhonabreul Sep 24, 2025
fe8dd81
Add Portfolio Turnover in optimization stats serialization
jhonabreul Oct 10, 2025
0fb02a7
Some cleanup
jhonabreul Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions Algorithm/QCAlgorithm.Plotting.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,12 @@ public void SetRuntimeStatistic(string name, double value)
[DocumentationAttribute(StatisticsTag)]
public void SetSummaryStatistic(string name, string value)
{
if (int.TryParse(name, NumberStyles.Integer, CultureInfo.InvariantCulture, out var intName) &&
intName >= 0 && intName <= 100)
{
throw new ArgumentException($"'{name}' is a reserved statistic name.");
}

_statisticsService.SetSummaryStatistic(name, value);
}

Expand All @@ -508,7 +514,7 @@ public void SetSummaryStatistic(string name, string value)
[DocumentationAttribute(StatisticsTag)]
public void SetSummaryStatistic(string name, int value)
{
_statisticsService.SetSummaryStatistic(name, value.ToStringInvariant());
SetSummaryStatistic(name, value.ToStringInvariant());
}

/// <summary>
Expand All @@ -519,7 +525,7 @@ public void SetSummaryStatistic(string name, int value)
[DocumentationAttribute(StatisticsTag)]
public void SetSummaryStatistic(string name, double value)
{
_statisticsService.SetSummaryStatistic(name, value.ToStringInvariant());
SetSummaryStatistic(name, value.ToStringInvariant());
}

/// <summary>
Expand All @@ -530,7 +536,7 @@ public void SetSummaryStatistic(string name, double value)
[DocumentationAttribute(StatisticsTag)]
public void SetSummaryStatistic(string name, decimal value)
{
_statisticsService.SetSummaryStatistic(name, value.ToStringInvariant());
SetSummaryStatistic(name, value.ToStringInvariant());
}

/// <summary>
Expand Down
114 changes: 73 additions & 41 deletions Common/Api/OptimizationBacktestJsonConverter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@

using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using QuantConnect.Optimizer.Parameters;
Expand All @@ -30,6 +30,39 @@ namespace QuantConnect.Api
/// </summary>
public class OptimizationBacktestJsonConverter : JsonConverter
{
private static string[] StatisticNames =
[
PerformanceMetrics.Alpha,
PerformanceMetrics.AnnualStandardDeviation,
PerformanceMetrics.AnnualVariance,
PerformanceMetrics.AverageLoss,
PerformanceMetrics.AverageWin,
PerformanceMetrics.Beta,
PerformanceMetrics.CompoundingAnnualReturn,
PerformanceMetrics.Drawdown,
PerformanceMetrics.EstimatedStrategyCapacity,
PerformanceMetrics.Expectancy,
PerformanceMetrics.InformationRatio,
PerformanceMetrics.LossRate,
PerformanceMetrics.NetProfit,
PerformanceMetrics.ProbabilisticSharpeRatio,
PerformanceMetrics.ProfitLossRatio,
PerformanceMetrics.SharpeRatio,
PerformanceMetrics.TotalFees,
PerformanceMetrics.TotalOrders,
PerformanceMetrics.TrackingError,
PerformanceMetrics.TreynorRatio,
PerformanceMetrics.WinRate,
PerformanceMetrics.SortinoRatio,
PerformanceMetrics.StartEquity,
PerformanceMetrics.EndEquity,
PerformanceMetrics.PortfolioTurnover,
PerformanceMetrics.DrawdownRecovery,
];

// Only 21 Lean statistics where supported when the serialized statistics where a json array
private static int ArrayStatisticsCount = 21;

/// <summary>
/// Determines whether this instance can convert the specified object type.
/// </summary>
Expand Down Expand Up @@ -97,25 +130,27 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
if (!optimizationBacktest.Statistics.IsNullOrEmpty())
{
writer.WritePropertyName("statistics");
writer.WriteStartArray();
foreach (var keyValuePair in optimizationBacktest.Statistics.OrderBy(pair => pair.Key))
writer.WriteStartObject();

var customStatisticsNames = new HashSet<string>();
int idx;
foreach (var (name, statisticValue, index) in optimizationBacktest.Statistics
.Select(kvp => (
Name: kvp.Key,
kvp.Value,
Index: (idx = Array.IndexOf(StatisticNames, kvp.Key)) != -1 ? idx : int.MaxValue
))
.OrderBy(t => t.Index)
.ThenByDescending(t => t.Name))
{
switch (keyValuePair.Key)
{
case PerformanceMetrics.PortfolioTurnover:
case PerformanceMetrics.SortinoRatio:
case PerformanceMetrics.StartEquity:
case PerformanceMetrics.EndEquity:
case PerformanceMetrics.DrawdownRecovery:
continue;
}
var statistic = keyValuePair.Value.Replace("%", string.Empty);
var statistic = statisticValue.Replace("%", string.Empty, StringComparison.InvariantCulture);
if (Currencies.TryParse(statistic, out var result))
{
writer.WritePropertyName(index < StatisticNames.Length? index.ToStringInvariant() : name);
writer.WriteValue(result);
}
}
writer.WriteEndArray();
writer.WriteEndObject();
}

if (optimizationBacktest.ParameterSet != null)
Expand Down Expand Up @@ -164,34 +199,25 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
Dictionary<string, string> statistics = default;
if (jStatistics != null)
{
statistics = new Dictionary<string, string>
if (jStatistics.Type == JTokenType.Array)
{
var statsCount = Math.Min(ArrayStatisticsCount, (jStatistics as JArray).Count);
statistics = new Dictionary<string, string>(StatisticNames
.Take(statsCount)
.Select((x, i) => KeyValuePair.Create(x, jStatistics[i].Value<string>()))
.Where(kvp => kvp.Value != null));
}
else
{
{ PerformanceMetrics.Alpha, jStatistics[0].Value<string>() },
{ PerformanceMetrics.AnnualStandardDeviation, jStatistics[1].Value<string>() },
{ PerformanceMetrics.AnnualVariance, jStatistics[2].Value<string>() },
{ PerformanceMetrics.AverageLoss, jStatistics[3].Value<string>() },
{ PerformanceMetrics.AverageWin, jStatistics[4].Value<string>() },
{ PerformanceMetrics.Beta, jStatistics[5].Value<string>() },
{ PerformanceMetrics.CompoundingAnnualReturn, jStatistics[6].Value<string>() },
{ PerformanceMetrics.Drawdown, jStatistics[7].Value<string>() },
{ PerformanceMetrics.EstimatedStrategyCapacity, jStatistics[8].Value<string>() },
{ PerformanceMetrics.Expectancy, jStatistics[9].Value<string>() },
{ PerformanceMetrics.InformationRatio, jStatistics[10].Value<string>() },
{ PerformanceMetrics.LossRate, jStatistics[11].Value<string>() },
{ PerformanceMetrics.NetProfit, jStatistics[12].Value<string>() },
{ PerformanceMetrics.ProbabilisticSharpeRatio, jStatistics[13].Value<string>() },
{ PerformanceMetrics.ProfitLossRatio, jStatistics[14].Value<string>() },
{ PerformanceMetrics.SharpeRatio, jStatistics[15].Value<string>() },
// TODO: Add SortinoRatio
// TODO: Add StartingEquity
// TODO: Add EndingEquity
// TODO: Add DrawdownRecovery
{ PerformanceMetrics.TotalFees, jStatistics[16].Value<string>() },
{ PerformanceMetrics.TotalOrders, jStatistics[17].Value<string>() },
{ PerformanceMetrics.TrackingError, jStatistics[18].Value<string>() },
{ PerformanceMetrics.TreynorRatio, jStatistics[19].Value<string>() },
{ PerformanceMetrics.WinRate, jStatistics[20].Value<string>() },
};
statistics = new();
foreach (var statistic in jStatistics.Children<JProperty>())
{
var statisticName = TryConvertToLeanStatisticIndex(statistic.Name, out var index)
? StatisticNames[index]
: statistic.Name;
statistics[statisticName] = statistic.Value.Value<string>();
}
}
}

var parameterSet = serializer.Deserialize<ParameterSet>(jObject["parameterSet"].CreateReader());
Expand Down Expand Up @@ -220,5 +246,11 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist

return optimizationBacktest;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryConvertToLeanStatisticIndex(string statistic, out int index)
{
return int.TryParse(statistic, out index) && index >= 0 && index < StatisticNames.Length;
}
}
}
11 changes: 11 additions & 0 deletions Tests/Algorithm/AlgorithmPlottingTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,5 +220,16 @@ public void PlotIndicatorPlotsBaseIndicator()
Assert.AreEqual("PlotTest", chart.Name);
Assert.AreEqual(sma1.Current.Value / sma2.Current.Value, chart.Series[ratio.Name].GetValues<ChartPoint>().First().y);
}

private static string[] ReservedSummaryStatisticNames => Enumerable.Range(0, 101).Select(i => i.ToStringInvariant()).ToArray();

[TestCaseSource(nameof(ReservedSummaryStatisticNames))]
public void ThrowsOnReservedSummaryStatisticName(string statisticName)
{
Assert.Throws<ArgumentException>(() => _algorithm.SetSummaryStatistic(statisticName, 0.1m));
Assert.Throws<ArgumentException>(() => _algorithm.SetSummaryStatistic(statisticName, 0.1));
Assert.Throws<ArgumentException>(() => _algorithm.SetSummaryStatistic(statisticName, 1));
Assert.Throws<ArgumentException>(() => _algorithm.SetSummaryStatistic(statisticName, "0.1"));
}
}
}
Loading
Loading