diff --git a/Common/Api/Optimization.cs b/Common/Api/Optimization.cs index d408f5d5e77c..cdfe5b50adc9 100644 --- a/Common/Api/Optimization.cs +++ b/Common/Api/Optimization.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using Newtonsoft.Json; +using QuantConnect.Optimizer; using QuantConnect.Optimizer.Objectives; using QuantConnect.Util; @@ -74,6 +75,12 @@ public class Optimization : BaseOptimization /// [JsonConverter(typeof(DateTimeJsonConverter), DateFormat.ISOShort, DateFormat.UI)] public DateTime Requested { get; set; } + + /// + /// Aggregate diagnostic of the optimization; omitted when no analysis was produced. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public OptimizationAnalysis Analysis { get; set; } } /// diff --git a/Common/Optimizer/Analysis/OptimizationAnalysisRunParameters.cs b/Common/Optimizer/Analysis/OptimizationAnalysisRunParameters.cs new file mode 100644 index 000000000000..378eafc21fbf --- /dev/null +++ b/Common/Optimizer/Analysis/OptimizationAnalysisRunParameters.cs @@ -0,0 +1,50 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Optimizer.Parameters; +using System.Collections.Generic; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// Bundles the inputs to the optimization analyzer: per-backtest metrics and the parameter grid spec. + /// + public class OptimizationAnalysisRunParameters + { + /// + /// Completed backtests from the optimization, already reduced to the metrics the analyzer reads. + /// + public IReadOnlyList CompletedBacktests { get; } + + /// + /// The optimization parameter grid spec. + /// + public IReadOnlyCollection OptimizationParameters { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The completed backtest metrics. + /// The parameter grid spec. + public OptimizationAnalysisRunParameters( + IReadOnlyList completedBacktests, + IReadOnlyCollection optimizationParameters) + { + CompletedBacktests = completedBacktests; + OptimizationParameters = optimizationParameters; + } + } +} diff --git a/Common/Optimizer/BacktestSummary.cs b/Common/Optimizer/BacktestSummary.cs new file mode 100644 index 000000000000..39247bda5b15 --- /dev/null +++ b/Common/Optimizer/BacktestSummary.cs @@ -0,0 +1,43 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// Per-backtest identity + Sharpe ratio shared by all optimization-analysis records that describe one backtest. + /// + public class BacktestSummary + { + /// + /// The backtest id; kept for programmatic access but not serialized into the analysis JSON. + /// + [JsonIgnore] + public string BacktestId { get; set; } + + /// + /// Parameter values the backtest was run with. + /// + public IReadOnlyDictionary Parameters { get; set; } + + /// + /// The backtest's Sharpe ratio. + /// + public decimal SharpeRatio { get; set; } + } +} diff --git a/Common/Optimizer/Cluster.cs b/Common/Optimizer/Cluster.cs new file mode 100644 index 000000000000..0f1d3bf38234 --- /dev/null +++ b/Common/Optimizer/Cluster.cs @@ -0,0 +1,56 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// One k-means cluster of backtests in standardized parameter space. + /// + public class Cluster + { + /// + /// Cluster centroid in original parameter units. + /// + public IReadOnlyDictionary Centroid { get; set; } + + /// + /// Number of backtests assigned to this cluster. + /// + public int MemberCount { get; set; } + + /// + /// Mean Sharpe ratio across the cluster's members. + /// + public decimal SharpeMean { get; set; } + + /// + /// Sample standard deviation of Sharpe ratios within this cluster. + /// + public decimal SharpeStdDev { get; set; } + + /// + /// Minimum Sharpe ratio within this cluster. + /// + public decimal SharpeMin { get; set; } + + /// + /// Maximum Sharpe ratio within this cluster. + /// + public decimal SharpeMax { get; set; } + } +} diff --git a/Common/Optimizer/FailedBacktestSummary.cs b/Common/Optimizer/FailedBacktestSummary.cs new file mode 100644 index 000000000000..661d3f83a770 --- /dev/null +++ b/Common/Optimizer/FailedBacktestSummary.cs @@ -0,0 +1,41 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// Breakdown of backtests in an optimization that produced zero orders. + /// + public class FailedBacktestSummary + { + /// + /// Total number of backtests that produced zero orders. + /// + public int ZeroOrderCount { get; set; } + + /// + /// Number of zero-order backtests inspected for analysis tags; may be smaller than . + /// + public int InspectedCount { get; set; } + + /// + /// Map of analysis-tag name to the number of inspected backtests carrying that tag. + /// + public IReadOnlyDictionary AnalysisNameCounts { get; set; } + } +} diff --git a/Common/Optimizer/LinearSegment.cs b/Common/Optimizer/LinearSegment.cs new file mode 100644 index 000000000000..e7e0b951d81b --- /dev/null +++ b/Common/Optimizer/LinearSegment.cs @@ -0,0 +1,44 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Optimizer +{ + /// + /// One linear piece of a piecewise interpolant on [, ], evaluated as y(x) = A + B * (x - XLo). + /// + public class LinearSegment + { + /// + /// Lower bound of this segment. + /// + public decimal XLo { get; set; } + + /// + /// Upper bound of this segment. + /// + public decimal XHi { get; set; } + + /// + /// Sharpe ratio at . + /// + public decimal A { get; set; } + + /// + /// Slope through the segment. + /// + public decimal B { get; set; } + } +} diff --git a/Common/Optimizer/Mode.cs b/Common/Optimizer/Mode.cs new file mode 100644 index 000000000000..4d40f43b862d --- /dev/null +++ b/Common/Optimizer/Mode.cs @@ -0,0 +1,29 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Optimizer +{ + /// + /// A local maximum of the Sharpe surface on the parameter grid; strictly greater than every face-neighbor's Sharpe. + /// + public class Mode : BacktestSummary + { + /// + /// Number of face-neighbors this backtest was compared against. + /// + public int NeighborCount { get; set; } + } +} diff --git a/Common/Optimizer/OptimizationAnalysis.cs b/Common/Optimizer/OptimizationAnalysis.cs new file mode 100644 index 000000000000..d26ca75b62ad --- /dev/null +++ b/Common/Optimizer/OptimizationAnalysis.cs @@ -0,0 +1,77 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using System.Collections.Generic; +using System.ComponentModel; + +namespace QuantConnect.Optimizer +{ + /// + /// Aggregate diagnostic produced by analyzing a completed optimization. + /// + public class OptimizationAnalysis + { + /// + /// Natural-language interpretation of the analysis produced by a downstream AI consumer; empty until populated. + /// + [DefaultValue("")] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Interpretation { get; set; } = string.Empty; + + /// + /// Total number of backtests observed, including failures. + /// + public int BacktestCountTotal { get; set; } + + /// + /// Number of backtests used in the analysis after filtering failures. + /// + public int BacktestCountUsed { get; set; } + + /// + /// Sharpe ratio statistics across all used backtests. + /// + public SharpeSummary OverallSharpe { get; set; } + + /// + /// The best-performing backtest (argmax of Sharpe). + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public BacktestSummary Best { get; set; } + + /// + /// Per-parameter sensitivity report; one entry per optimized parameter. + /// + public IReadOnlyList Parameters { get; set; } + + /// + /// K-means clusters in standardized parameter space, ordered by mean Sharpe descending. + /// + public IReadOnlyList Clusters { get; set; } + + /// + /// Local maxima of the Sharpe surface on the parameter grid, ordered by Sharpe descending. + /// + public IReadOnlyList Modes { get; set; } + + /// + /// Breakdown of zero-order backtests; null when none exist. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public FailedBacktestSummary FailedBacktests { get; set; } + } +} diff --git a/Common/Optimizer/OptimizationBacktestMetrics.cs b/Common/Optimizer/OptimizationBacktestMetrics.cs new file mode 100644 index 000000000000..7ee81492c08c --- /dev/null +++ b/Common/Optimizer/OptimizationBacktestMetrics.cs @@ -0,0 +1,115 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using QuantConnect.Logging; +using QuantConnect.Optimizer.Parameters; +using QuantConnect.Packets; +using QuantConnect.Statistics; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace QuantConnect.Optimizer +{ + /// + /// Lightweight per-backtest record extracted at time to avoid retaining the full backtest JSON. + /// + public class OptimizationBacktestMetrics : BacktestSummary + { + /// + /// The backtest's total performance (wraps , + /// , and list); null when absent from the backtest result. + /// + public AlgorithmPerformance TotalPerformance { get; set; } + + /// + /// Number of orders the backtest produced. + /// + public int TotalOrders { get; set; } + + /// + /// Names of the diagnostic entries the backtest attached. + /// + public IReadOnlyList AnalysisNames { get; set; } + + /// + /// Extracts the fields the analyzer needs from a backtest result JSON; returns null when the parameter set is invalid. + /// + /// The backtest id. + /// The parameter set the backtest was run with. + /// The serialized backtest result JSON. + public static OptimizationBacktestMetrics ExtractFrom(string backtestId, ParameterSet parameterSet, string jsonBacktestResult) + { + if (parameterSet == null) + { + return null; + } + + var parameters = ParseParameterSet(parameterSet); + if (parameters.Count == 0) + { + return null; + } + + BacktestResult parsed = null; + if (!string.IsNullOrEmpty(jsonBacktestResult)) + { + try + { + // DeserializeJson uses the LEAN-wide JsonSerializer (CamelCaseNamingStrategy + OrderJsonConverter), + // so polymorphic Orders and camelCase JSON both work without extra configuration. + parsed = jsonBacktestResult.DeserializeJson(); + } + catch (JsonException ex) + { + Log.Error(ex, $"OptimizationBacktestMetrics.ExtractFrom(): failed to parse backtest result for '{backtestId}'"); + } + } + + var analysisNames = parsed?.Analysis == null + ? (IReadOnlyList)System.Array.Empty() + : parsed.Analysis + .Where(a => !string.IsNullOrEmpty(a?.Name)) + .Select(a => a.Name) + .ToList(); + + return new OptimizationBacktestMetrics + { + BacktestId = backtestId, + Parameters = parameters, + SharpeRatio = parsed?.TotalPerformance?.PortfolioStatistics?.SharpeRatio ?? 0m, + TotalPerformance = parsed?.TotalPerformance, + TotalOrders = parsed?.Orders?.Count ?? 0, + AnalysisNames = analysisNames + }; + } + + private static Dictionary ParseParameterSet(ParameterSet parameterSet) + { + var result = new Dictionary(); + if (parameterSet?.Value == null) return result; + foreach (var kv in parameterSet.Value) + { + if (decimal.TryParse(kv.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) + { + result[kv.Key] = d; + } + } + return result; + } + } +} diff --git a/Common/Optimizer/ParameterReport.cs b/Common/Optimizer/ParameterReport.cs new file mode 100644 index 000000000000..a33b286772f7 --- /dev/null +++ b/Common/Optimizer/ParameterReport.cs @@ -0,0 +1,78 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// Sensitivity report for a single optimized parameter. + /// + public class ParameterReport + { + /// + /// Parameter name. + /// + public string Name { get; set; } + + /// + /// Lower bound of the parameter sweep. + /// + public decimal SearchedMin { get; set; } + + /// + /// Upper bound of the parameter sweep. + /// + public decimal SearchedMax { get; set; } + + /// + /// Sweep step size; null when not provided in the optimization configuration. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public decimal? Step { get; set; } + + /// + /// Mean Sharpe range (max - min) across every 1-D slice. + /// + public decimal MeanWithinSliceSharpeRange { get; set; } + + /// + /// Maximum Sharpe range (max - min) across every 1-D slice. + /// + public decimal MaxWithinSliceSharpeRange { get; set; } + + /// + /// Worst-case Sharpe change between two adjacent grid values, scaled by . + /// + public decimal MaxAbsDerivativePerStep { get; set; } + + /// + /// This parameter's value at the best backtest. + /// + public decimal BestValue { get; set; } + + /// + /// True when lies within half a step of or . + /// + public bool BestAtSearchedEdge { get; set; } + + /// + /// One-dimensional slices used for the sensitivity analysis. + /// + public IReadOnlyList Slices { get; set; } + } +} diff --git a/Common/Optimizer/SharpeSummary.cs b/Common/Optimizer/SharpeSummary.cs new file mode 100644 index 000000000000..0ddaf4b7fc00 --- /dev/null +++ b/Common/Optimizer/SharpeSummary.cs @@ -0,0 +1,49 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Optimizer +{ + /// + /// Sharpe ratio statistics across all used backtests in an optimization. + /// + public class SharpeSummary + { + /// + /// Arithmetic mean of Sharpe ratios. + /// + public decimal Mean { get; set; } + + /// + /// Sample standard deviation of Sharpe ratios. + /// + public decimal StdDev { get; set; } + + /// + /// Minimum Sharpe ratio observed. + /// + public decimal Min { get; set; } + + /// + /// Maximum Sharpe ratio observed. + /// + public decimal Max { get; set; } + + /// + /// Median Sharpe ratio. + /// + public decimal Median { get; set; } + } +} diff --git a/Common/Optimizer/SliceFit.cs b/Common/Optimizer/SliceFit.cs new file mode 100644 index 000000000000..742e2d32eafe --- /dev/null +++ b/Common/Optimizer/SliceFit.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// One-dimensional cross-section of the parameter space: one parameter varies while every other is held constant. + /// + public class SliceFit + { + /// + /// Values of the other parameters held constant for this slice. + /// + public IReadOnlyDictionary FixedParameters { get; set; } + + /// + /// Max Sharpe minus min Sharpe across this slice. + /// + public decimal SharpeRange { get; set; } + + /// + /// Maximum absolute slope across this slice's linear segments. + /// + public decimal MaxAbsDerivative { get; set; } + + /// + /// Piecewise linear pieces of the fit; one per adjacent pair of grid points. + /// + public IReadOnlyList Segments { get; set; } + } +} diff --git a/Optimizer/Analysis/OptimizationAnalyzer.cs b/Optimizer/Analysis/OptimizationAnalyzer.cs new file mode 100644 index 000000000000..6d8755bc5dbd --- /dev/null +++ b/Optimizer/Analysis/OptimizationAnalyzer.cs @@ -0,0 +1,103 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Logging; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// Builds an aggregate from a completed optimization's per-backtest metrics; optimization-side analogue of the Engine ResultsAnalyzer. + /// + public class OptimizationAnalyzer + { + /// + /// Runs the full optimization-analysis pipeline. + /// + /// Completed backtest metrics plus the parameter grid spec. + /// The populated , or null when no usable backtests remain. + public OptimizationAnalysis Run(OptimizationAnalysisRunParameters parameters) + { + var allBacktests = parameters?.CompletedBacktests ?? new List(); + var backtests = allBacktests.Where(b => b?.TotalPerformance?.PortfolioStatistics != null).ToList(); + if (backtests.Count == 0) + { + Log.Trace("OptimizationAnalyzer.Run(): no completed backtests with parsable Sharpe ratios; skipping analysis"); + return null; + } + + var sharpes = backtests.Select(b => b.SharpeRatio).ToList(); + var overall = new SharpeSummary + { + Mean = sharpes.Average(), + StdDev = StdDev(sharpes), + Min = sharpes.Min(), + Max = sharpes.Max(), + Median = Median(sharpes) + }; + + // Sharpe is the universal yardstick regardless of the optimization's Criterion. + var best = backtests.OrderByDescending(b => b.SharpeRatio).First(); + var bestSummary = new BacktestSummary + { + BacktestId = best.BacktestId, + Parameters = new Dictionary(best.Parameters), + SharpeRatio = best.SharpeRatio + }; + + var paramReports = parameters.OptimizationParameters + .Select(p => OptimizationSlicing.AnalyzeParameter(p, backtests, best)) + .ToList(); + + var clusters = OptimizationClustering.Build(backtests, parameters.OptimizationParameters); + var modes = OptimizationModes.Find(backtests, parameters.OptimizationParameters); + var failed = OptimizationFailedBacktests.Build(allBacktests); + + return new OptimizationAnalysis + { + BacktestCountTotal = allBacktests.Count, + BacktestCountUsed = backtests.Count, + OverallSharpe = overall, + Best = bestSummary, + Parameters = paramReports, + Clusters = clusters, + Modes = modes, + FailedBacktests = failed + }; + } + + // ── Aggregate helpers ──────────────────────────────────────────────────── + + private static decimal StdDev(IReadOnlyCollection values) + { + if (values.Count < 2) return 0m; + var mean = values.Average(); + var s = values.Sum(v => (v - mean) * (v - mean)); + // System.Math has no decimal Sqrt; cross into double for the root and back. + return (decimal)System.Math.Sqrt((double)(s / (values.Count - 1))); + } + + private static decimal Median(IEnumerable values) + { + var sorted = values.OrderBy(v => v).ToList(); + if (sorted.Count == 0) return 0m; + return sorted.Count % 2 == 1 + ? sorted[sorted.Count / 2] + : 0.5m * (sorted[sorted.Count / 2 - 1] + sorted[sorted.Count / 2]); + } + } +} diff --git a/Optimizer/Analysis/OptimizationClustering.cs b/Optimizer/Analysis/OptimizationClustering.cs new file mode 100644 index 000000000000..928ca0e401cc --- /dev/null +++ b/Optimizer/Analysis/OptimizationClustering.cs @@ -0,0 +1,259 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Optimizer.Parameters; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// K-means clustering of backtests in standardized parameter space with k chosen by an elbow heuristic. + /// + internal static class OptimizationClustering + { + private const int KMin = 2; + private const int KMaxAbsolute = 5; + private const int Seed = 42; + private const int MaxIterations = 100; + private const double PlateauThreshold = 0.7; + + public static IReadOnlyList Build( + IReadOnlyList backtests, + IReadOnlyCollection parameters) + { + var output = new List(); + if (backtests == null || parameters == null) return output; + if (backtests.Count < KMin + 1 || parameters.Count == 0) return output; + + var paramNames = parameters.Select(p => p.Name).ToArray(); + + var usable = backtests + .Where(b => paramNames.All(b.Parameters.ContainsKey)) + .ToList(); + if (usable.Count < KMin + 1) return output; + + // Cap k_max at ceil(sqrt(N)) so small N doesn't get carved into too many clusters. + var sqrtCap = (int)Math.Ceiling(Math.Sqrt(usable.Count)); + var kMaxEffective = Math.Min(KMaxAbsolute, sqrtCap); + var maxK = Math.Min(kMaxEffective, usable.Count - 1); + if (maxK < KMin) return output; + + // K-means math runs in double so we can use Math.Sqrt / distance comparisons; + // we convert back to decimal at the boundary for the centroid output. + var raw = usable + .Select(b => paramNames.Select(n => (double)b.Parameters[n]).ToArray()) + .ToArray(); + var (normalized, means, stds) = Standardize(raw); + + var byK = new Dictionary(); + for (var k = KMin; k <= maxK; k++) + { + byK[k] = KMeans(normalized, k); + } + var bestK = SelectKByElbow(byK); + var pick = byK[bestK]; + + var centroidsOriginal = pick.Centroids + .Select(c => Denormalize(c, means, stds)) + .ToArray(); + + for (var c = 0; c < bestK; c++) + { + var memberIndices = Enumerable.Range(0, usable.Count) + .Where(i => pick.Labels[i] == c) + .ToList(); + if (memberIndices.Count == 0) continue; + var sharpes = memberIndices.Select(i => usable[i].SharpeRatio).ToList(); + + var centroidDict = new Dictionary(paramNames.Length); + for (var d = 0; d < paramNames.Length; d++) + { + centroidDict[paramNames[d]] = (decimal)centroidsOriginal[c][d]; + } + + output.Add(new Cluster + { + Centroid = centroidDict, + MemberCount = memberIndices.Count, + SharpeMean = sharpes.Average(), + SharpeStdDev = StdDev(sharpes), + SharpeMin = sharpes.Min(), + SharpeMax = sharpes.Max() + }); + } + + // Order by mean Sharpe descending so the best-performing region is first. + return output.OrderByDescending(x => x.SharpeMean).ToList(); + } + + private static int SelectKByElbow(Dictionary results) + { + var ks = results.Keys.OrderBy(k => k).ToList(); + if (ks.Count == 1) return ks[0]; + for (var i = 1; i < ks.Count; i++) + { + var prev = results[ks[i - 1]].Wcss; + var curr = results[ks[i]].Wcss; + if (prev > 0 && curr / prev > PlateauThreshold) return ks[i - 1]; + } + return ks[^1]; + } + + private sealed class KMeansResult + { + public int[] Labels { get; } + public double[][] Centroids { get; } + public double Wcss { get; } + + public KMeansResult(int[] labels, double[][] centroids, double wcss) + { + Labels = labels; + Centroids = centroids; + Wcss = wcss; + } + } + + private static KMeansResult KMeans(double[][] points, int k) + { + var n = points.Length; + var d = points[0].Length; + var rng = new Random(Seed); + + // k-means++ initialization. + var centroids = new double[k][]; + centroids[0] = (double[])points[rng.Next(n)].Clone(); + for (var c = 1; c < k; c++) + { + var dists = new double[n]; + for (var i = 0; i < n; i++) + { + var min = double.MaxValue; + for (var j = 0; j < c; j++) + { + var dd = SquaredDistance(points[i], centroids[j]); + if (dd < min) min = dd; + } + dists[i] = min; + } + var sum = dists.Sum(); + var pick = rng.NextDouble() * sum; + double acc = 0; + var chosen = n - 1; + for (var i = 0; i < n; i++) + { + acc += dists[i]; + if (acc >= pick) { chosen = i; break; } + } + centroids[c] = (double[])points[chosen].Clone(); + } + + // Lloyd's iteration. + var labels = new int[n]; + for (var iter = 0; iter < MaxIterations; iter++) + { + var changed = false; + for (var i = 0; i < n; i++) + { + var best = 0; + var bestDist = double.MaxValue; + for (var c = 0; c < k; c++) + { + var dd = SquaredDistance(points[i], centroids[c]); + if (dd < bestDist) { bestDist = dd; best = c; } + } + if (labels[i] != best) { labels[i] = best; changed = true; } + } + if (!changed && iter > 0) break; + + var sums = new double[k][]; + var counts = new int[k]; + for (var c = 0; c < k; c++) sums[c] = new double[d]; + for (var i = 0; i < n; i++) + { + var c = labels[i]; + counts[c]++; + for (var j = 0; j < d; j++) sums[c][j] += points[i][j]; + } + for (var c = 0; c < k; c++) + { + if (counts[c] == 0) continue; + for (var j = 0; j < d; j++) centroids[c][j] = sums[c][j] / counts[c]; + } + } + + double wcss = 0; + for (var i = 0; i < n; i++) wcss += SquaredDistance(points[i], centroids[labels[i]]); + return new KMeansResult(labels, centroids, wcss); + } + + private static (double[][] Normalized, double[] Means, double[] Stds) Standardize(double[][] points) + { + var n = points.Length; + var d = points[0].Length; + var means = new double[d]; + var stds = new double[d]; + for (var j = 0; j < d; j++) + { + double s = 0; + for (var i = 0; i < n; i++) s += points[i][j]; + means[j] = s / n; + } + for (var j = 0; j < d; j++) + { + double s = 0; + for (var i = 0; i < n; i++) + { + var t = points[i][j] - means[j]; + s += t * t; + } + stds[j] = n > 1 ? Math.Sqrt(s / (n - 1)) : 1.0; + if (stds[j] < 1e-12) stds[j] = 1.0; + } + var normalized = new double[n][]; + for (var i = 0; i < n; i++) + { + normalized[i] = new double[d]; + for (var j = 0; j < d; j++) normalized[i][j] = (points[i][j] - means[j]) / stds[j]; + } + return (normalized, means, stds); + } + + private static double[] Denormalize(double[] standardized, double[] means, double[] stds) + { + var d = standardized.Length; + var result = new double[d]; + for (var j = 0; j < d; j++) result[j] = standardized[j] * stds[j] + means[j]; + return result; + } + + private static double SquaredDistance(double[] a, double[] b) + { + double s = 0; + for (var i = 0; i < a.Length; i++) { var d = a[i] - b[i]; s += d * d; } + return s; + } + + private static decimal StdDev(IReadOnlyCollection values) + { + if (values.Count < 2) return 0m; + var mean = values.Average(); + var s = values.Sum(v => (v - mean) * (v - mean)); + return (decimal)Math.Sqrt((double)(s / (values.Count - 1))); + } + } +} diff --git a/Optimizer/Analysis/OptimizationFailedBacktests.cs b/Optimizer/Analysis/OptimizationFailedBacktests.cs new file mode 100644 index 000000000000..63b8f65fe507 --- /dev/null +++ b/Optimizer/Analysis/OptimizationFailedBacktests.cs @@ -0,0 +1,64 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// Counts how many zero-order backtests carry each backtest-level analysis tag; returns null when no zero-order backtests exist. + /// + internal static class OptimizationFailedBacktests + { + // Cap on inspected backtests; a rough tally is enough. + private const int MaxBacktestsToInspect = 10; + + public static FailedBacktestSummary Build(IReadOnlyList backtests) + { + if (backtests == null) return null; + + var zeroOrder = backtests.Where(b => b.TotalOrders == 0).ToList(); + if (zeroOrder.Count == 0) return null; + + var sample = zeroOrder.Take(MaxBacktestsToInspect).ToList(); + var nameCount = new Dictionary(StringComparer.Ordinal); + + foreach (var backtest in sample) + { + // De-dupe per backtest so counts are "backtests carrying the tag", not raw occurrences. + var seen = new HashSet(StringComparer.Ordinal); + if (backtest.AnalysisNames != null) + { + foreach (var name in backtest.AnalysisNames) + { + if (string.IsNullOrEmpty(name)) continue; + if (!seen.Add(name)) continue; + nameCount[name] = nameCount.GetValueOrDefault(name, 0) + 1; + } + } + } + + return new FailedBacktestSummary + { + ZeroOrderCount = zeroOrder.Count, + InspectedCount = sample.Count, + AnalysisNameCounts = nameCount + }; + } + } +} diff --git a/Optimizer/Analysis/OptimizationModes.cs b/Optimizer/Analysis/OptimizationModes.cs new file mode 100644 index 000000000000..859227e9345d --- /dev/null +++ b/Optimizer/Analysis/OptimizationModes.cs @@ -0,0 +1,108 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Optimizer.Parameters; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// Detects local maxima of the Sharpe surface; backtests strictly greater than every face-neighbor on the parameter grid. + /// + internal static class OptimizationModes + { + public static IReadOnlyList Find( + IReadOnlyList backtests, + IReadOnlyCollection parameters) + { + var modes = new List(); + if (backtests == null || parameters == null) return modes; + if (parameters.Count == 0 || backtests.Count == 0) return modes; + + var paramNames = parameters.Select(p => p.Name).ToArray(); + + // Sorted distinct values per parameter define the grid axes. + var axisValues = new Dictionary>(); + foreach (var name in paramNames) + { + axisValues[name] = backtests + .Where(b => b.Parameters.ContainsKey(name)) + .Select(b => b.Parameters[name]) + .Distinct() + .OrderBy(v => v) + .ToList(); + } + + // Map each backtest to its grid position. + var indexed = new List<(OptimizationBacktestMetrics Backtest, int[] Indices)>(); + foreach (var b in backtests) + { + if (!paramNames.All(b.Parameters.ContainsKey)) continue; + var idx = new int[paramNames.Length]; + var ok = true; + for (var d = 0; d < paramNames.Length; d++) + { + idx[d] = axisValues[paramNames[d]].IndexOf(b.Parameters[paramNames[d]]); + if (idx[d] < 0) { ok = false; break; } + } + if (ok) indexed.Add((b, idx)); + } + + var byTuple = indexed.ToDictionary(p => TupleKey(p.Indices), p => p.Backtest); + + foreach (var (backtest, idx) in indexed) + { + var totalNeighbors = 0; + var dominatesAll = true; + + for (var d = 0; d < paramNames.Length && dominatesAll; d++) + { + var axisLen = axisValues[paramNames[d]].Count; + foreach (var delta in new[] { -1, 1 }) + { + var ni = idx[d] + delta; + if (ni < 0 || ni >= axisLen) continue; + + var neighborIdx = (int[])idx.Clone(); + neighborIdx[d] = ni; + if (!byTuple.TryGetValue(TupleKey(neighborIdx), out var neighbor)) continue; + + totalNeighbors++; + if (neighbor.SharpeRatio >= backtest.SharpeRatio) { dominatesAll = false; break; } + } + } + + if (dominatesAll && totalNeighbors > 0) + { + modes.Add(new Mode + { + BacktestId = backtest.BacktestId, + Parameters = new Dictionary(backtest.Parameters), + SharpeRatio = backtest.SharpeRatio, + NeighborCount = totalNeighbors + }); + } + } + + return modes.OrderByDescending(m => m.SharpeRatio).ToList(); + } + + private static string TupleKey(int[] indices) + => string.Join(",", indices.Select(i => i.ToString(CultureInfo.InvariantCulture))); + } +} diff --git a/Optimizer/Analysis/OptimizationSlicing.cs b/Optimizer/Analysis/OptimizationSlicing.cs new file mode 100644 index 000000000000..cf1283ab6859 --- /dev/null +++ b/Optimizer/Analysis/OptimizationSlicing.cs @@ -0,0 +1,173 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Optimizer.Parameters; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// Per-parameter sensitivity analysis via 1-D slices through the backtest cloud with a piecewise linear fit. + /// + internal static class OptimizationSlicing + { + public static ParameterReport AnalyzeParameter( + OptimizationParameter parameter, + IReadOnlyList backtests, + OptimizationBacktestMetrics best) + { + var name = parameter.Name; + var owning = backtests.Where(b => b.Parameters.ContainsKey(name)).ToList(); + + var otherParamNames = owning + .SelectMany(b => b.Parameters.Keys) + .Where(k => k != name) + .Distinct() + .OrderBy(k => k, StringComparer.Ordinal) + .ToList(); + + // Group backtests by other-parameter values; each group is one 1-D slice. + IEnumerable> grouped = otherParamNames.Count == 0 + ? new[] { owning.GroupBy(_ => "").FirstOrDefault() } + .Where(g => g != null) + .Cast>() + : owning.GroupBy(b => SliceKey(b, otherParamNames)); + + var slices = new List(); + foreach (var group in grouped) + { + var slice = BuildSlice(group.ToList(), name, otherParamNames); + if (slice != null) slices.Add(slice); + } + + var hasBest = best.Parameters.TryGetValue(name, out var bestValue); + var (searchedMin, searchedMax, step) = ExtractGridSpec(parameter, owning, name); + var bestAtEdge = hasBest && IsAtSearchedEdge(bestValue, searchedMin, searchedMax, step); + + var meanRange = slices.Count > 0 ? slices.Average(s => s.SharpeRange) : 0m; + var maxRange = slices.Count > 0 ? slices.Max(s => s.SharpeRange) : 0m; + var maxDerivPerStep = slices.Count > 0 + ? slices.Max(s => s.MaxAbsDerivative) * (step ?? 1m) + : 0m; + + return new ParameterReport + { + Name = name, + SearchedMin = searchedMin, + SearchedMax = searchedMax, + Step = step, + MeanWithinSliceSharpeRange = meanRange, + MaxWithinSliceSharpeRange = maxRange, + MaxAbsDerivativePerStep = maxDerivPerStep, + BestValue = bestValue, + BestAtSearchedEdge = bestAtEdge, + Slices = slices + }; + } + + private static SliceFit BuildSlice( + List backtests, + string varyingParamName, + IReadOnlyList otherParamNames) + { + // Defensively collapse duplicate parameter values by averaging Sharpes. + var points = backtests + .GroupBy(b => b.Parameters[varyingParamName]) + .Select(g => (X: g.Key, Y: g.Average(b => b.SharpeRatio))) + .OrderBy(p => p.X) + .ToList(); + + if (points.Count == 0) return null; + + var xs = points.Select(p => p.X).ToList(); + var ys = points.Select(p => p.Y).ToList(); + var sharpeRange = ys.Count >= 2 ? ys.Max() - ys.Min() : 0m; + + // Piecewise linear: one segment per adjacent pair; slope is sensitivity per parameter unit. + var segments = new List(); + decimal maxAbsDerivative = 0m; + for (var i = 0; i < points.Count - 1; i++) + { + var dx = xs[i + 1] - xs[i]; + var slope = (ys[i + 1] - ys[i]) / dx; + segments.Add(new LinearSegment + { + XLo = xs[i], + XHi = xs[i + 1], + A = ys[i], + B = slope + }); + var absSlope = Math.Abs(slope); + if (absSlope > maxAbsDerivative) maxAbsDerivative = absSlope; + } + + var fixedParams = new Dictionary(); + if (otherParamNames.Count > 0) + { + var first = backtests[0]; + foreach (var p in otherParamNames) + { + if (first.Parameters.TryGetValue(p, out var v)) fixedParams[p] = v; + } + } + + return new SliceFit + { + FixedParameters = fixedParams, + SharpeRange = sharpeRange, + MaxAbsDerivative = maxAbsDerivative, + Segments = segments + }; + } + + private static (decimal Min, decimal Max, decimal? Step) ExtractGridSpec( + OptimizationParameter parameter, + IReadOnlyList owning, + string name) + { + if (parameter is OptimizationStepParameter step) + { + return (step.MinValue, step.MaxValue, step.Step); + } + + // Fallback for non-step parameters: infer min/max/step from measured values. + var values = owning.Select(b => b.Parameters[name]).Distinct().OrderBy(v => v).ToList(); + if (values.Count == 0) return (0m, 0m, null); + if (values.Count == 1) return (values[0], values[0], null); + + var min = values[0]; + var max = values[^1]; + var gaps = new List(); + for (var i = 1; i < values.Count; i++) gaps.Add(values[i] - values[i - 1]); + return (min, max, gaps.Min()); + } + + private static bool IsAtSearchedEdge(decimal value, decimal min, decimal max, decimal? step) + { + var tol = ((step ?? 1m) / 2m) + 1e-9m; + return Math.Abs(value - min) <= tol || Math.Abs(value - max) <= tol; + } + + private static string SliceKey(OptimizationBacktestMetrics backtest, IReadOnlyList otherParamNames) + { + return string.Join("|", otherParamNames.Select(p => + (backtest.Parameters.TryGetValue(p, out var v) ? v.ToString(CultureInfo.InvariantCulture) : "NaN"))); + } + } +} diff --git a/Optimizer/LeanOptimizer.cs b/Optimizer/LeanOptimizer.cs index 3912a54daaa6..28bac18bf537 100644 --- a/Optimizer/LeanOptimizer.cs +++ b/Optimizer/LeanOptimizer.cs @@ -21,6 +21,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; +using QuantConnect.Optimizer.Analysis; using QuantConnect.Optimizer.Objectives; using QuantConnect.Optimizer.Parameters; using QuantConnect.Optimizer.Strategies; @@ -41,6 +42,9 @@ public abstract class LeanOptimizer : IDisposable private int _completedBacktest; private volatile bool _disposed; + // Per-backtest metrics extracted in NewResult so we don't retain the full backtest JSON. + private readonly List _completedBacktests = new(); + /// /// The total completed backtests count /// @@ -183,6 +187,29 @@ protected virtual void TriggerOnEndEvent() // we clean up before we send an update so that the runtime stats are updated CleanUpRunningInstance(); + + // Set Analysis before ProcessUpdate so SendUpdate can upload it; guarded so analyzer failure never breaks the optimization. + if (result != null) + { + try + { + // Snapshot under the lock so a late NewResult on another thread can't mutate the list mid-enumeration. + List backtestsSnapshot; + lock (_completedBacktests) + { + backtestsSnapshot = new List(_completedBacktests); + } + var parameters = new OptimizationAnalysisRunParameters( + backtestsSnapshot, + NodePacket.OptimizationParameters); + result.Analysis = new OptimizationAnalyzer().Run(parameters); + } + catch (Exception ex) + { + Log.Error(ex, "Error running optimization analysis"); + } + } + ProcessUpdate(forceSend: true); Ended?.Invoke(this, result); @@ -246,6 +273,17 @@ protected virtual void NewResult(string jsonBacktestResult, string backtestId) result = new OptimizationResult(jsonBacktestResult, parameterSet, backtestId); } + // Extract metrics now and drop the heavy JSON; null results (invalid parameters) are skipped. + var metrics = OptimizationBacktestMetrics.ExtractFrom(backtestId, parameterSet, jsonBacktestResult); + if (metrics != null) + { + // Backtest results can arrive on different threads; guard _completedBacktests with its own lock. + lock (_completedBacktests) + { + _completedBacktests.Add(metrics); + } + } + // always notify the strategy Strategy.PushNewResults(result); diff --git a/Optimizer/OptimizationResult.cs b/Optimizer/OptimizationResult.cs index 74412b73b3b6..7d7e97e1e376 100644 --- a/Optimizer/OptimizationResult.cs +++ b/Optimizer/OptimizationResult.cs @@ -47,6 +47,11 @@ public class OptimizationResult /// public ParameterSet ParameterSet { get; } + /// + /// Aggregate diagnostic for the whole optimization; populated only on the final result fired via . + /// + public OptimizationAnalysis Analysis { get; set; } + /// /// Create an instance of /// diff --git a/Tests/Optimizer/Analysis/LeanOptimizerAnalysisTests.cs b/Tests/Optimizer/Analysis/LeanOptimizerAnalysisTests.cs new file mode 100644 index 000000000000..9b0a0a209b4b --- /dev/null +++ b/Tests/Optimizer/Analysis/LeanOptimizerAnalysisTests.cs @@ -0,0 +1,150 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using NUnit.Framework; +using QuantConnect.Optimizer; +using QuantConnect.Optimizer.Objectives; +using QuantConnect.Optimizer.Parameters; +using QuantConnect.Orders; +using QuantConnect.Packets; +using QuantConnect.Statistics; +using QuantConnect.Util; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace QuantConnect.Tests.Optimizer.Analysis +{ + /// + /// End-to-end tests for 's analyzer wiring via the event. + /// + [TestFixture, Parallelizable(ParallelScope.Self)] + public class LeanOptimizerAnalysisTests + { + [Test] + public void Ended_AttachesAnalysis_WhenBacktestsCarrySharpeRatios() + { + using var resetEvent = new ManualResetEvent(false); + var packet = new OptimizationNodePacket + { + Criterion = new Target("Profit", new Maximization(), null), + OptimizationParameters = new HashSet + { + new OptimizationStepParameter("x", 1, 4, 1), + new OptimizationStepParameter("y", 10, 40, 10) + }, + MaximumConcurrentBacktests = 8 + }; + using var optimizer = new SharpeEmittingFakeLeanOptimizer(packet); + + OptimizationResult result = null; + optimizer.Ended += (s, solution) => + { + result = solution; + optimizer.DisposeSafely(); + resetEvent.Set(); + }; + + optimizer.Start(); + resetEvent.WaitOne(); + + Assert.NotNull(result); + Assert.NotNull(result.Analysis, "Analysis should be populated when backtests have Sharpe ratios"); + Assert.Greater(result.Analysis.BacktestCountUsed, 0); + Assert.NotNull(result.Analysis.Best); + Assert.NotNull(result.Analysis.OverallSharpe); + Assert.AreEqual(2, result.Analysis.Parameters.Count); + } + + [Test] + public void Ended_LeavesAnalysisNull_WhenNoBacktestCarriesSharpe() + { + // FakeLeanOptimizer's payload carries no Sharpe; analyzer must safely skip. + using var resetEvent = new ManualResetEvent(false); + var packet = new OptimizationNodePacket + { + Criterion = new Target("Profit", new Maximization(), null), + OptimizationParameters = new HashSet + { + new OptimizationStepParameter("ema-slow", 1, 5, 1), + new OptimizationStepParameter("ema-fast", 10, 50, 10) + }, + MaximumConcurrentBacktests = 8 + }; + using var optimizer = new FakeLeanOptimizer(packet); + + OptimizationResult result = null; + optimizer.Ended += (s, solution) => + { + result = solution; + optimizer.DisposeSafely(); + resetEvent.Set(); + }; + + optimizer.Start(); + resetEvent.WaitOne(); + + Assert.NotNull(result, "Ended must still fire even with no analyzable backtests"); + Assert.IsNull(result.Analysis, "Analysis should be null when no backtest carries a Sharpe ratio"); + } + + /// + /// fake that emits backtest JSON shaped like a real one, with a deterministic Sharpe. + /// + private sealed class SharpeEmittingFakeLeanOptimizer : LeanOptimizer + { + public SharpeEmittingFakeLeanOptimizer(OptimizationNodePacket nodePacket) : base(nodePacket) + { + } + + protected override string RunLean(ParameterSet parameterSet, string backtestName) + { + var id = Guid.NewGuid().ToString(); + Task.Delay(10).ContinueWith(_ => + { + var x = parameterSet.Value.TryGetValue("x", out var xs) && decimal.TryParse(xs, NumberStyles.Any, CultureInfo.InvariantCulture, out var xv) ? xv : 0m; + var y = parameterSet.Value.TryGetValue("y", out var ys) && decimal.TryParse(ys, NumberStyles.Any, CultureInfo.InvariantCulture, out var yv) ? yv : 0m; + // Math.Pow is double-only; cross into double for the surface and back. + var sharpe = (decimal)(1.0 - 0.05 * Math.Pow((double)x - 3, 2) - 0.0005 * Math.Pow((double)y - 25, 2)); + // Build a real BacktestResult and serialize via the LEAN-wide JsonSerializer + // so the JSON shape matches what BacktestingResultHandler produces. + var result = new QuantConnect.Packets.BacktestResult + { + // Statistics dict is what the optimizer's Criterion targets (e.g. "Statistics.Profit"). + Statistics = new Dictionary + { + ["Profit"] = (x + y).ToString(CultureInfo.InvariantCulture) + }, + // Typed TotalPerformance.PortfolioStatistics is what the analyzer reads. + TotalPerformance = new AlgorithmPerformance(), + Orders = Enumerable.Range(1, 10).ToDictionary(i => i, i => (Order)new MarketOrder()), + Analysis = Array.Empty() + }; + result.TotalPerformance.PortfolioStatistics.SharpeRatio = sharpe; + NewResult(result.SerializeJsonToString(), id); + }); + return id; + } + + protected override void AbortLean(string backtestId) { } + protected override void SendUpdate() { } + } + } +} diff --git a/Tests/Optimizer/Analysis/OptimizationAnalyzerTests.cs b/Tests/Optimizer/Analysis/OptimizationAnalyzerTests.cs new file mode 100644 index 000000000000..35d365defa88 --- /dev/null +++ b/Tests/Optimizer/Analysis/OptimizationAnalyzerTests.cs @@ -0,0 +1,232 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using NUnit.Framework; +using QuantConnect.Optimizer; +using QuantConnect.Optimizer.Analysis; +using QuantConnect.Optimizer.Parameters; +using QuantConnect.Orders; +using QuantConnect.Packets; +using QuantConnect.Statistics; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace QuantConnect.Tests.Optimizer.Analysis +{ + [TestFixture, Parallelizable(ParallelScope.Self)] + public class OptimizationAnalyzerTests + { + [Test] + public void Run_ProducesOverallSharpeStats() + { + // 3x3 grid of synthetic Sharpe values. + var sharpes = new decimal[,] + { + { 0.10m, 0.20m, 0.30m }, + { 0.15m, 0.25m, 0.35m }, + { 0.18m, 0.28m, 0.38m } + }; + + var backtests = BuildGridBacktests(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 3, yCount: 3); + var analyzer = new OptimizationAnalyzer(); + + var analysis = analyzer.Run(new OptimizationAnalysisRunParameters(backtests, parameters)); + + Assert.NotNull(analysis); + Assert.AreEqual(9, analysis.BacktestCountUsed); + Assert.AreEqual(9, analysis.BacktestCountTotal); + + // Mean = average of {0.10..0.38}. + Assert.That(analysis.OverallSharpe.Mean, Is.EqualTo(0.2433m).Within(0.001m)); + Assert.AreEqual(0.10m, analysis.OverallSharpe.Min); + Assert.AreEqual(0.38m, analysis.OverallSharpe.Max); + } + + [Test] + public void Run_BestBacktestIsArgmaxSharpe() + { + var sharpes = new decimal[,] + { + { 0.10m, 0.20m, 0.30m }, + { 0.15m, 0.25m, 0.35m }, + { 0.18m, 0.28m, 0.99m } // peak at (2, 2) + }; + + var backtests = BuildGridBacktests(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 3, yCount: 3); + var analysis = new OptimizationAnalyzer().Run(new OptimizationAnalysisRunParameters(backtests, parameters)); + + Assert.NotNull(analysis.Best); + Assert.AreEqual(0.99m, analysis.Best.SharpeRatio); + // Parameters at (xIndex=2, yIndex=2). Grid x: {1,2,3}; y: {10,20,30}. + Assert.AreEqual(3m, analysis.Best.Parameters["x"]); + Assert.AreEqual(30m, analysis.Best.Parameters["y"]); + } + + [Test] + public void Run_FindsInteriorMode() + { + // 3x3 with a single interior peak at (1, 1): should produce one mode with 4 neighbors. + var sharpes = new decimal[,] + { + { 0.10m, 0.20m, 0.10m }, + { 0.20m, 0.99m, 0.20m }, + { 0.10m, 0.20m, 0.10m } + }; + + var backtests = BuildGridBacktests(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 3, yCount: 3); + var analysis = new OptimizationAnalyzer().Run(new OptimizationAnalysisRunParameters(backtests, parameters)); + + Assert.AreEqual(1, analysis.Modes.Count); + Assert.AreEqual(0.99m, analysis.Modes[0].SharpeRatio); + Assert.AreEqual(4, analysis.Modes[0].NeighborCount); + } + + [Test] + public void Run_ClusterCountRespectsSqrtCap() + { + // 4 backtests -> ceil(sqrt(4)) = 2 -> max 2 clusters. + var sharpes = new decimal[,] + { + { 0.10m, 0.20m }, + { 0.30m, 0.40m } + }; + + var backtests = BuildGridBacktests(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 2, yCount: 2); + var analysis = new OptimizationAnalyzer().Run(new OptimizationAnalysisRunParameters(backtests, parameters)); + + Assert.LessOrEqual(analysis.Clusters.Count, 2); + } + + [Test] + public void Run_BuildsFailedBacktestSummary_FromZeroOrderBacktests() + { + // 2x2 grid; every backtest has zero orders and carries known analysis tags. + var sharpes = new decimal[,] + { + { 0m, 0m }, + { 0m, 0m } + }; + + var backtests = BuildGridBacktests( + sharpes, + totalOrders: 0, + analysisNames: new[] { "FlatEquityCurveAnalysis", "ExecutionSpeedAnalysis" }); + var parameters = BuildGridParameters(xCount: 2, yCount: 2); + var analysis = new OptimizationAnalyzer().Run(new OptimizationAnalysisRunParameters(backtests, parameters)); + + Assert.NotNull(analysis.FailedBacktests); + Assert.AreEqual(4, analysis.FailedBacktests.ZeroOrderCount); + Assert.AreEqual(4, analysis.FailedBacktests.InspectedCount); + Assert.AreEqual(4, analysis.FailedBacktests.AnalysisNameCounts["FlatEquityCurveAnalysis"]); + Assert.AreEqual(4, analysis.FailedBacktests.AnalysisNameCounts["ExecutionSpeedAnalysis"]); + } + + [Test] + public void Run_OmitsFailedBacktestSummary_WhenAllBacktestsTrade() + { + var sharpes = new decimal[,] + { + { 0.10m, 0.20m }, + { 0.30m, 0.40m } + }; + + var backtests = BuildGridBacktests(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 2, yCount: 2); + var analysis = new OptimizationAnalyzer().Run(new OptimizationAnalysisRunParameters(backtests, parameters)); + + Assert.IsNull(analysis.FailedBacktests); + } + + [Test] + public void ExtractFrom_ParsesSharpeAndAnalysisNamesFromBacktestJson() + { + var parameterSet = new ParameterSet(0, new Dictionary { ["x"] = "1", ["y"] = "10" }); + var json = BuildBacktestJson(0.75m, totalOrders: 12, new[] { "FlatEquityCurveAnalysis" }); + + var metrics = OptimizationBacktestMetrics.ExtractFrom("bt-0", parameterSet, json); + + Assert.NotNull(metrics); + Assert.NotNull(metrics.TotalPerformance?.PortfolioStatistics); + Assert.AreEqual(0.75m, metrics.SharpeRatio); + Assert.AreEqual(0.75m, metrics.TotalPerformance.PortfolioStatistics.SharpeRatio); + Assert.AreEqual(12, metrics.TotalOrders); + CollectionAssert.AreEqual(new[] { "FlatEquityCurveAnalysis" }, metrics.AnalysisNames.ToArray()); + Assert.AreEqual(1m, metrics.Parameters["x"]); + Assert.AreEqual(10m, metrics.Parameters["y"]); + } + + // ── helpers ────────────────────────────────────────────────────────────── + + private static List BuildGridBacktests( + decimal[,] sharpes, + int totalOrders, + string[] analysisNames = null) + { + var result = new List(); + var xCount = sharpes.GetLength(0); + var yCount = sharpes.GetLength(1); + var id = 0; + for (var i = 0; i < xCount; i++) + { + for (var j = 0; j < yCount; j++) + { + var paramSet = new ParameterSet(id, new Dictionary + { + ["x"] = (i + 1).ToString(CultureInfo.InvariantCulture), + ["y"] = ((j + 1) * 10).ToString(CultureInfo.InvariantCulture) + }); + var json = BuildBacktestJson(sharpes[i, j], totalOrders, analysisNames); + result.Add(OptimizationBacktestMetrics.ExtractFrom($"backtest-{id}", paramSet, json)); + id++; + } + } + return result; + } + + private static string BuildBacktestJson(decimal sharpe, int totalOrders, string[] analysisNames) + { + // Build a real BacktestResult and serialize through the LEAN-wide JsonSerializer + // (CamelCaseNamingStrategy) so the JSON shape matches what BacktestingResultHandler + // produces in production — which is what OptimizationBacktestMetrics.ExtractFrom + // round-trips through DeserializeJson. + var result = new QuantConnect.Packets.BacktestResult + { + TotalPerformance = new AlgorithmPerformance(), + Orders = Enumerable.Range(1, totalOrders).ToDictionary(i => i, i => (Order)new MarketOrder()), + Analysis = (analysisNames ?? System.Array.Empty()) + .Select(n => new QuantConnect.Analysis(n, "issue", null, null, System.Array.Empty())) + .ToList() + }; + result.TotalPerformance.PortfolioStatistics.SharpeRatio = sharpe; + return result.SerializeJsonToString(); + } + + private static HashSet BuildGridParameters(int xCount, int yCount) + { + return new HashSet + { + new OptimizationStepParameter("x", 1, xCount, 1), + new OptimizationStepParameter("y", 10, yCount * 10, 10) + }; + } + } +}