From cd19e787f80e90a22b7a39aab82d05504b5cac11 Mon Sep 17 00:00:00 2001 From: Peter Dekkers Date: Sun, 23 Jul 2023 09:35:26 +0200 Subject: [PATCH] Further enhancements back test --- .../main/kotlin/org/roboquant/Roboquant.kt | 25 ++-- .../kotlin/org/roboquant/backtest/Backtest.kt | 40 ++---- .../org/roboquant/backtest/Optimizer.kt | 133 +++++++----------- .../kotlin/org/roboquant/backtest/Params.kt | 8 +- .../kotlin/org/roboquant/backtest/Score.kt | 10 +- .../org/roboquant/backtest/SearchSpace.kt | 42 +++--- .../kotlin/org/roboquant/common/TimeSpan.kt | 2 + .../org/roboquant/loggers/LastEntryLogger.kt | 17 ++- .../kotlin/org/roboquant/RoboquantTest.kt | 2 +- .../org/roboquant/backtest/OptimizerTest.kt | 26 +++- .../org/roboquant/backtest/SearchSpaceTest.kt | 8 +- .../roboquant/loggers/LastEntryLoggerTest.kt | 6 +- 12 files changed, 153 insertions(+), 166 deletions(-) diff --git a/roboquant/src/main/kotlin/org/roboquant/Roboquant.kt b/roboquant/src/main/kotlin/org/roboquant/Roboquant.kt index 15c2fac3c..933cdf361 100644 --- a/roboquant/src/main/kotlin/org/roboquant/Roboquant.kt +++ b/roboquant/src/main/kotlin/org/roboquant/Roboquant.kt @@ -23,14 +23,15 @@ import org.roboquant.brokers.Broker import org.roboquant.brokers.closeSizes import org.roboquant.brokers.sim.SimBroker import org.roboquant.common.Logging +import org.roboquant.common.TimeSpan import org.roboquant.common.Timeframe +import org.roboquant.common.plus import org.roboquant.feeds.Event import org.roboquant.feeds.EventChannel import org.roboquant.feeds.Feed import org.roboquant.feeds.TradePrice import org.roboquant.loggers.MemoryLogger import org.roboquant.loggers.MetricsLogger -import org.roboquant.loggers.SilentLogger import org.roboquant.metrics.Metric import org.roboquant.orders.MarketOrder import org.roboquant.orders.createCancelOrders @@ -110,26 +111,23 @@ data class Roboquant( } } - /** - * Returns a copy of this instance but with the [SilentLogger] enabled. - * - * This is typically used during warmup and training phases when metrics output are not of interest. - */ - fun silent() = copy(logger = SilentLogger()) /** * Start a new run using the provided [feed] as data. If no [timeframe] is provided all the events in the feed * will be processed. You can provide a custom [name] that will help to later identify this run. If none is * provided, a name will be generated with the format "run-" + * Additionally you can provide a [warmup] period in which no metrics will be logged or orders placed. + * * This is the synchronous (blocking) method of run that is convenient to use. However, if you want to execute runs * in parallel have a look at [runAsync] */ fun run( feed: Feed, timeframe: Timeframe = feed.timeframe, - name: String = "run", + warmup: TimeSpan = TimeSpan.ZERO, + name: String = "run" ) = runBlocking { - runAsync(feed, timeframe, name) + runAsync(feed, timeframe, warmup, name) } /** @@ -142,7 +140,8 @@ data class Roboquant( suspend fun runAsync( feed: Feed, timeframe: Timeframe = feed.timeframe, - name: String = "run", + warmup: TimeSpan = TimeSpan.ZERO, + name: String = "run" ) { val channel = EventChannel(channelCapacity, timeframe) val scope = CoroutineScope(Dispatchers.Default + Job()) @@ -152,6 +151,8 @@ data class Roboquant( } start(name, timeframe) + val warmupEnd = timeframe.start + warmup + try { while (true) { val event = channel.receive() @@ -161,12 +162,12 @@ data class Roboquant( broker.sync(event) val account = broker.account val metricResult = getMetrics(account, event) - logger.log(metricResult, time, name) + if (time >= warmupEnd) logger.log(metricResult, time, name) // Generate signals and place orders val signals = strategy.generate(event) val orders = policy.act(signals, account, event) - broker.place(orders, time) + if (time >= warmupEnd) broker.place(orders, time) kotlinLogger.trace { "time=$${event.time} actions=${event.actions.size} signals=${signals.size} orders=${orders.size}" diff --git a/roboquant/src/main/kotlin/org/roboquant/backtest/Backtest.kt b/roboquant/src/main/kotlin/org/roboquant/backtest/Backtest.kt index f30a0fa97..1c00bd2e0 100644 --- a/roboquant/src/main/kotlin/org/roboquant/backtest/Backtest.kt +++ b/roboquant/src/main/kotlin/org/roboquant/backtest/Backtest.kt @@ -20,13 +20,17 @@ import org.roboquant.Roboquant import org.roboquant.brokers.sim.SimBroker import org.roboquant.common.TimeSpan import org.roboquant.common.Timeframe -import org.roboquant.common.plus import org.roboquant.feeds.Feed /** - * Run back test without parameter optimizations. This is useful to get insights into the performance over different - * timeframes. + * Run back tests without parameter optimizations. + * The back tests in this class will be run sequentially. + * + * These types of back tests are especially useful to get insights into the performance over different timeframes. + * + * @property feed The feed to use during the back tests + * @property roboquant The roboquant instance to use during the back tests */ class Backtest(val feed: Feed, val roboquant: Roboquant) { @@ -34,30 +38,12 @@ class Backtest(val feed: Feed, val roboquant: Roboquant) { require(roboquant.broker is SimBroker) { "Only a SimBroker can be used for back testing"} } - - private val broker - get() = roboquant.broker as SimBroker - - - /** - * Run a warmup and return the remaining timeframe - */ - private fun warmup(timeframe: Timeframe, period: TimeSpan) : Timeframe { - require(timeframe.isFinite()) - val end = timeframe.start + period - val tf = Timeframe(timeframe.start, end) - roboquant.silent().run(feed, tf) // Run without logging - broker.reset() // Reset the broker after a warmup - return timeframe.copy(start = end) - } - /** * Perform a single run over the provided [timeframe] using the provided [warmup] period. The timeframe is * including the warmup period. */ fun singleRun(timeframe: Timeframe = feed.timeframe, warmup: TimeSpan = TimeSpan.ZERO) { - val tf = if (! warmup.isZero) warmup(timeframe, warmup) else timeframe - roboquant.run(feed, tf, name = "run-$tf") + roboquant.run(feed, timeframe, warmup, name = "run-$timeframe") } /** @@ -70,8 +56,7 @@ class Backtest(val feed: Feed, val roboquant: Roboquant) { ) { require(feed.timeframe.isFinite()) { "feed needs a finite timeframe" } feed.timeframe.split(period, warmup).forEach { - val tf = if (! warmup.isZero) warmup(it, warmup) else it - roboquant.run(feed, tf, name = "run-$tf") + roboquant.run(feed, it, warmup, name = "run-$it") roboquant.reset(false) } } @@ -80,7 +65,7 @@ class Backtest(val feed: Feed, val roboquant: Roboquant) { * Run a Monte Carlo simulation of a number of [samples] to find out how metrics of interest are distributed * given the provided [period]. * - * Optional each period can be preceded by a [warmup] time-span. + * Optional each period can include a [warmup] time-span. */ fun monteCarlo( period: TimeSpan, @@ -88,9 +73,8 @@ class Backtest(val feed: Feed, val roboquant: Roboquant) { warmup: TimeSpan = TimeSpan.ZERO, ) { require(feed.timeframe.isFinite()) { "feed needs a finite timeframe" } - feed.timeframe.sample(period + warmup, samples).forEach { - val tf = if (! warmup.isZero) warmup(it, warmup) else it - roboquant.run(feed, tf, name = "run-$tf") + feed.timeframe.sample(period, samples).forEach { + roboquant.run(feed, it, warmup, name = "run-$it") roboquant.reset(false) } } diff --git a/roboquant/src/main/kotlin/org/roboquant/backtest/Optimizer.kt b/roboquant/src/main/kotlin/org/roboquant/backtest/Optimizer.kt index d1df1b129..c670e4f11 100644 --- a/roboquant/src/main/kotlin/org/roboquant/backtest/Optimizer.kt +++ b/roboquant/src/main/kotlin/org/roboquant/backtest/Optimizer.kt @@ -1,12 +1,13 @@ package org.roboquant.backtest import org.roboquant.Roboquant +import org.roboquant.brokers.sim.SimBroker import org.roboquant.common.ParallelJobs import org.roboquant.common.TimeSpan import org.roboquant.common.Timeframe +import org.roboquant.common.minus import org.roboquant.feeds.Feed -import org.roboquant.loggers.LastEntryLogger -import org.roboquant.loggers.MetricsLogger +import org.roboquant.loggers.MemoryLogger import java.util.* data class RunResult(val params: Params, val score: Double, val timeframe: Timeframe, val name: String) @@ -16,74 +17,87 @@ data class RunResult(val params: Params, val score: Double, val timeframe: Timef */ fun mutableSynchronisedListOf(): MutableList = Collections.synchronizedList(mutableListOf()) - /** - * Default optimizer that implements default back-test optimization strategies to find a set of optimal parameter + * Optimizer that implements different back-test optimization strategies to find a set of optimal parameter * values. * + * An optimizing back test has two phases, and each phase has up to two periods. + * The warmup periods are optional and by default [TimeSpan.ZERO]. + * + * Training phase: + * - warmup period; get required data for strategies, policies and metrics loaded + * - training period; optimize the hyperparameters + * + * Validation phase + * - warmup period; get required data for strategies, policies and metrics loaded + * - validation period; see how a run is performing, based on unseen data + * + * * @property space search space * @property score scoring function - * @property trainLogger the metrics logger to use for training phases, default is [LastEntryLogger] * @property getRoboquant function that returns an instance of roboquant based on passed parameters * */ -class Optimizer( +open class Optimizer( private val space: SearchSpace, private val score: Score, - private val trainLogger: MetricsLogger = LastEntryLogger(), private val getRoboquant: (Params) -> Roboquant ) { private var run = 0 - /** * Using the default objective to maximize a metric. The default objective will use the last entry of the * provided [evalMetric] as the value to optimize. */ constructor(space: SearchSpace, evalMetric: String, getRoboquant: (Params) -> Roboquant) : this( - space, MetricScore(evalMetric), LastEntryLogger(), getRoboquant + space, MetricScore(evalMetric), getRoboquant ) - fun walkForward(feed: Feed, period: TimeSpan, anchored: Boolean = false): List { + fun walkForward( + feed: Feed, + period: TimeSpan, + warmup: TimeSpan = TimeSpan.ZERO, + anchored: Boolean = false + ): List { require(!feed.timeframe.isInfinite()) { "feed needs known timeframe" } val start = feed.timeframe.start val results = mutableListOf() feed.timeframe.split(period).forEach { val timeframe = if (anchored) Timeframe(start, it.end, it.inclusive) else it - val result = train(feed, timeframe) + val result = train(feed, timeframe, warmup) results.addAll(result) } return results } - - /* - * Walk-forward with validation (out of sample) - + /** + * Run a walk forward + */ fun walkForward( feed: Feed, period: TimeSpan, validation: TimeSpan, - warmup: TimeSpan = 0.days, + warmup: TimeSpan = TimeSpan.ZERO, anchored: Boolean = false ): List { - require(!feed.timeframe.isInfinite()) { "feed needs known timeframe" } + val timeframe = feed.timeframe + require(timeframe.isFinite()) { "feed needs known timeframe" } val feedStart = feed.timeframe.start val results = mutableListOf() - feed.timeframe.split(period + validation, period).forEach { + timeframe.split(period + validation, period).forEach { val trainEnd = it.end - validation val trainStart = if (anchored) feedStart else it.start val trainTimeframe = Timeframe(trainStart, trainEnd) - val result = train(feed, trainTimeframe) + val result = train(feed, trainTimeframe, warmup) results.addAll(result) val best = result.maxBy { entry -> entry.score } // println("phase=training timeframe=$timeframe equity=${best.second} params=${best.first}") - val validationTimeframe = Timeframe(trainEnd, it.end - warmup, it.inclusive) - val score = validate(feed, validationTimeframe, best.params) + val validationTimeframe = Timeframe(trainEnd, it.end, it.inclusive) + val score = validate(feed, validationTimeframe, best.params, warmup) results.add(score) } return results @@ -91,45 +105,14 @@ class Optimizer( - fun walkForward2( - feed: Feed, - period: TimeSpan, - validation: TimeSpan, - warmup: TimeSpan? = null, - anchored: Boolean = false - ) { - val start = feed.timeframe.start - feed.timeframe.split(period).forEach { - - val endTrain = it.end - validation - val tf = if (anchored) Timeframe(start, endTrain) else it - - val result = train(feed, tf) - val best = result.maxBy { r -> r.score } - println("timeframe=$tf equity=${best.score} params=${best.params}") - val tf2 = Timeframe(endTrain, it.end, it.inclusive) - val rq = getRoboquant(best.params) - if (warmup != null) { - val warmupTimeframe = Timeframe(endTrain - warmup, endTrain, false) - rq.warmup(feed, warmupTimeframe) - } - - rq.run(feed, tf2) - val validationScore = score.calculate(rq, tf2) - } - - } - - */ - /** * Run a Monte Carlo simulation */ - fun monteCarlo(feed: Feed, period: TimeSpan, samples: Int): List { + fun monteCarlo(feed: Feed, period: TimeSpan, samples: Int, warmup: TimeSpan = TimeSpan.ZERO): List { val results = mutableSynchronisedListOf() require(!feed.timeframe.isInfinite()) { "feed needs known timeframe" } feed.timeframe.sample(period, samples).forEach { - val result = train(feed, it) + val result = train(feed, it, warmup) results.addAll(result) // val best = result.maxBy { it.score } // println("timeframe=$it equity=${best.score} params=${best.params}") @@ -137,17 +120,24 @@ class Optimizer( return results } + /** + * The logger to use for training phase. By default, this logger is discarded after the run and score is + * calculated + */ + protected fun getTrainLogger() = MemoryLogger(false) + /** * Train the solution in parallel */ - fun train(feed: Feed, tf: Timeframe = Timeframe.INFINITE): List { + fun train(feed: Feed, tf: Timeframe = Timeframe.INFINITE, warmup: TimeSpan = TimeSpan.ZERO): List { val jobs = ParallelJobs() val results = mutableSynchronisedListOf() - for (params in space.materialize()) { + for (params in space) { jobs.add { - val rq = getRoboquant(params) + val rq = getRoboquant(params).copy(logger = getTrainLogger()) + require(rq.broker is SimBroker) { "Only a SimBroker can be used for back testing"} val name = "train-${run++}" - rq.copy(logger = trainLogger).runAsync(feed, tf, name = name) + rq.runAsync(feed, tf, warmup, name = name) val s = score.calculate(rq, name, tf) val result = RunResult(params, s, tf, name) results.add(result) @@ -158,34 +148,17 @@ class Optimizer( return results } - /** - * Train the solution in parallel - - fun train2(feed: Feed, tf: Timeframe): List { - val results = mutableListOf() - for (params in space.materialize()) { - // println("running roboquant timeframe=$tf params=$params") - val name = "train-${run++}" - val roboquant = getRoboquant(params) - roboquant.run(feed, tf, name = name) - val s = score.calculate(roboquant, tf) - val result = RunResult(params, s, tf, name) - // println("phase=train result=$result") - results.add(result) - } - return results - } - - private fun validate(feed: Feed, timeframe: Timeframe, params: Params): RunResult { + private fun validate(feed: Feed, timeframe: Timeframe, params: Params, warmup: TimeSpan): RunResult { val rq = getRoboquant(params) + require(rq.broker is SimBroker) { "Only a SimBroker can be used for back testing"} + val name = "validate-${run++}" - rq.run(feed, timeframe, name = name) - val s = score.calculate(rq, timeframe) + rq.run(feed, timeframe, warmup, name = name) + val s = score.calculate(rq, name, timeframe) // println("phase=validation result=$result") return RunResult(params, s, timeframe, name) } - */ } \ No newline at end of file diff --git a/roboquant/src/main/kotlin/org/roboquant/backtest/Params.kt b/roboquant/src/main/kotlin/org/roboquant/backtest/Params.kt index 1e9d18f40..95597a1b1 100644 --- a/roboquant/src/main/kotlin/org/roboquant/backtest/Params.kt +++ b/roboquant/src/main/kotlin/org/roboquant/backtest/Params.kt @@ -18,22 +18,22 @@ package org.roboquant.backtest /** - * Holds the (hyper-)parameters that can be used in back tests that try to find the optimum set of parameters. + * Holds the (hyper-)parameters that can be used in a [SearchSpace] to define that search space. */ class Params : LinkedHashMap() { /** - * Returns the string value for the provided parameter [name] + * Returns the [String] value for the provided parameter [name] */ fun getString(name: String) = get(name) as String /** - * Returns the int value for the provided parameter [name] + * Returns the [Int] value for the provided parameter [name] */ fun getInt(name: String) = get(name) as Int /** - * Returns the double value for the provided parameter [name] + * Returns the [Double] value for the provided parameter [name] */ fun getDouble(name: String) = get(name) as Double diff --git a/roboquant/src/main/kotlin/org/roboquant/backtest/Score.kt b/roboquant/src/main/kotlin/org/roboquant/backtest/Score.kt index 1e6bac997..413f7d346 100644 --- a/roboquant/src/main/kotlin/org/roboquant/backtest/Score.kt +++ b/roboquant/src/main/kotlin/org/roboquant/backtest/Score.kt @@ -35,7 +35,9 @@ fun interface Score { /** * The annualized equity growth as a [Score] function. - * This function works best for longer timeframe runs (over 3 months). + * + * This function works best for longer timeframe runs (over 3 months). Otherwise, calculated values can have + * large fluctuations. */ fun annualizedEquityGrowth(roboquant: Roboquant, timeframe: Timeframe): Double { val broker = roboquant.broker as SimBroker @@ -50,11 +52,15 @@ fun interface Score { /** * Use the last value of recorded metric name as the score. + * + * Use this with a metrics logger that supports the retrieval of metrics. */ class MetricScore(private val metricName: String) : Score { override fun calculate(roboquant: Roboquant, run: String, timeframe: Timeframe): Double { - val metrics = roboquant.logger.getMetric(metricName)[run] ?: return Double.NaN + val logger = roboquant.logger + // assert(logger.metricNames.contains(metricName)) { "$metricName not found in logger" } + val metrics = logger.getMetric(metricName)[run] ?: return Double.NaN return metrics.values.last() } diff --git a/roboquant/src/main/kotlin/org/roboquant/backtest/SearchSpace.kt b/roboquant/src/main/kotlin/org/roboquant/backtest/SearchSpace.kt index b696e53d9..1d91c0765 100644 --- a/roboquant/src/main/kotlin/org/roboquant/backtest/SearchSpace.kt +++ b/roboquant/src/main/kotlin/org/roboquant/backtest/SearchSpace.kt @@ -20,12 +20,8 @@ package org.roboquant.backtest /** * Interface for all types of search spaces */ -interface SearchSpace { +interface SearchSpace : Iterable { - /** - * Returns an [Iterable] over the parameters defined in this search space. - */ - fun materialize() : Iterable /** * Update the search space based on an observation. @@ -47,12 +43,12 @@ interface SearchSpace { /** * If you just want to run a certain type of back test without any hyperparameter search, you can use this search space. - * It contains a single entry with no parameters, so iterating over this search space will run exactly once. + * It contains a single entry with no parameters, so iterating over this search space will run a back test exactly once. */ class EmptySearchSpace : SearchSpace { - override fun materialize(): Iterable { - return listOf(Params()).asIterable() + override fun iterator(): Iterator { + return listOf(Params()).listIterator() } /** @@ -67,6 +63,10 @@ class EmptySearchSpace : SearchSpace { /** * Random Search Space * + * In Random Search, we try random combinations of the values of the [Params] and evaluate the trading strategy + * for these selected combination. + * The number of combinations is limited to [size]. + * * @property size the total number of samples that will be drawn from this random search space */ class RandomSearch(override val size: Int) : SearchSpace { @@ -96,11 +96,12 @@ class RandomSearch(override val size: Int) : SearchSpace { } - override fun materialize(): List { - return list + override fun iterator(): Iterator { + return list.listIterator() } + /** * Add a parameter function */ @@ -120,7 +121,11 @@ class RandomSearch(override val size: Int) : SearchSpace { } /** - * Create a Grid Search Space + * Create a Grid Search Space. + * + * In Grid Search, we try every combination of the values of the [Params] and evaluate the trading strategy for + * each combination + * */ class GridSearch : SearchSpace { @@ -155,17 +160,6 @@ class GridSearch : SearchSpace { params[name] = values.toList() } - /* - fun nextRandomSample() : Params { - val result = Params() - for ((key, values) in params) { - val idx = Config.random.nextInt(values.lastIndex) - result[key] = values[idx] - } - return result - } - */ - fun add(name: String, samples: Int, fn: () -> Any) { params[name] = (1..samples).map { fn() } @@ -175,8 +169,8 @@ class GridSearch : SearchSpace { override val size get() = params.map { it.value.size }.reduce(Int::times) - override fun materialize(): List { - return list + override fun iterator(): Iterator { + return list.listIterator() } } diff --git a/roboquant/src/main/kotlin/org/roboquant/common/TimeSpan.kt b/roboquant/src/main/kotlin/org/roboquant/common/TimeSpan.kt index 3ab35e1bf..156f6bc49 100644 --- a/roboquant/src/main/kotlin/org/roboquant/common/TimeSpan.kt +++ b/roboquant/src/main/kotlin/org/roboquant/common/TimeSpan.kt @@ -191,6 +191,7 @@ val Long.nanos */ fun Instant.plus(period: TimeSpan, zoneId: ZoneId): Instant { // Optimized path for HFT + if (period == TimeSpan.ZERO) return this val result = if (period.period == Period.ZERO) this else atZone(zoneId).plus(period.period).toInstant() return result.plus(period.duration) } @@ -200,6 +201,7 @@ fun Instant.plus(period: TimeSpan, zoneId: ZoneId): Instant { */ fun Instant.minus(period: TimeSpan, zoneId: ZoneId): Instant { // Optimized path for HFT + if (period == TimeSpan.ZERO) return this val result = if (period.period == Period.ZERO) this else atZone(zoneId).minus(period.period).toInstant() return result.minus(period.duration) } diff --git a/roboquant/src/main/kotlin/org/roboquant/loggers/LastEntryLogger.kt b/roboquant/src/main/kotlin/org/roboquant/loggers/LastEntryLogger.kt index af8d15ed5..41b7a241a 100644 --- a/roboquant/src/main/kotlin/org/roboquant/loggers/LastEntryLogger.kt +++ b/roboquant/src/main/kotlin/org/roboquant/loggers/LastEntryLogger.kt @@ -20,6 +20,12 @@ import org.roboquant.common.Observation import org.roboquant.common.TimeSeries import org.roboquant.common.Timeframe import java.time.Instant +import java.util.* + +/** + * Create a mutable synchronized list + */ +// fun mutableSynchronisedMapOf(): MutableMap = Collections.synchronizedMap(mutableMapOf()) /** * Stores the last value of a metric for a particular run in memory. @@ -33,17 +39,19 @@ import java.time.Instant class LastEntryLogger(var showProgress: Boolean = false) : MetricsLogger { // The key is runName - private val history = mutableMapOf>() + private val history = Hashtable>() private val progressBar = ProgressBar() @Synchronized override fun log(results: Map, time: Instant, run: String) { if (showProgress) progressBar.update(time) - for ((t, u) in results) { + if (results.isNotEmpty()) { val map = history.getOrPut(run) { mutableMapOf() } - val value = Observation(time, u) - map[t] = value + for ((t, u) in results) { + val value = Observation(time, u) + map[t] = value + } } } @@ -72,6 +80,7 @@ class LastEntryLogger(var showProgress: Boolean = false) : MetricsLogger { /** * Get results for the metric specified by its [name]. */ + @Synchronized override fun getMetric(name: String): Map { val result = mutableMapOf>() for ((run, metrics) in history) { diff --git a/roboquant/src/test/kotlin/org/roboquant/RoboquantTest.kt b/roboquant/src/test/kotlin/org/roboquant/RoboquantTest.kt index 7ba66b1fb..84e6523a6 100644 --- a/roboquant/src/test/kotlin/org/roboquant/RoboquantTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/RoboquantTest.kt @@ -131,7 +131,7 @@ internal class RoboquantTest { val broker = SimBroker(initial) val logger = MemoryLogger() val roboquant = Roboquant(strategy, ProgressMetric(), broker = broker, logger = logger) - roboquant.silent().run(feed, Timeframe.INFINITE) + roboquant.copy(logger = SilentLogger()).run(feed, Timeframe.INFINITE) roboquant.broker.reset() assertTrue(logger.history.isEmpty()) val account = broker.account diff --git a/roboquant/src/test/kotlin/org/roboquant/backtest/OptimizerTest.kt b/roboquant/src/test/kotlin/org/roboquant/backtest/OptimizerTest.kt index 050e1c1da..4879f8ef6 100644 --- a/roboquant/src/test/kotlin/org/roboquant/backtest/OptimizerTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/backtest/OptimizerTest.kt @@ -27,10 +27,10 @@ class OptimizerTest { val feed = RandomWalkFeed.lastYears(1, nAssets = 1) - val r1 = opt.train(feed) + val r1 = opt.train(feed, feed.timeframe) assertTrue(r1.isNotEmpty()) - val r2 = opt.walkForward(feed, 6.months, false) + val r2 = opt.walkForward(feed, 6.months) assertTrue(r2.isNotEmpty()) val r3 = opt.monteCarlo(feed, 6.months, 5) @@ -38,6 +38,26 @@ class OptimizerTest { } + @Test + fun complete() { + val space = GridSearch() + space.add("x", 3..15) + space.add("y", 2..10) + + val logger = LastEntryLogger() + val opt = Optimizer(space, "account.equity") { params -> + val x = params.getInt("x") + val y = x + params.getInt("y") + val s = EMAStrategy(x, y) + Roboquant(s, AccountMetric(), logger = logger) + } + + val feed = RandomWalkFeed.lastYears(3, nAssets = 2) + val r2 = opt.walkForward(feed, 9.months, 3.months, 0.months, false) + println(r2) + + } + @Test fun noParams() { @@ -47,7 +67,7 @@ class OptimizerTest { } val feed = RandomWalkFeed.lastYears(1, nAssets = 1) - val r1 = opt.train(feed) + val r1 = opt.train(feed, feed.timeframe) assertTrue(r1.isNotEmpty()) } diff --git a/roboquant/src/test/kotlin/org/roboquant/backtest/SearchSpaceTest.kt b/roboquant/src/test/kotlin/org/roboquant/backtest/SearchSpaceTest.kt index b74452960..ec2722e6a 100644 --- a/roboquant/src/test/kotlin/org/roboquant/backtest/SearchSpaceTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/backtest/SearchSpaceTest.kt @@ -16,7 +16,7 @@ class SearchSpaceTest { Config.random.nextInt() } - val params = space.materialize() + val params = space.iterator().asSequence().toList() assertEquals(100, params.size) assertTrue(params.all { it.contains("p1") }) assertTrue(params.all { it.contains("p2") }) @@ -32,7 +32,7 @@ class SearchSpaceTest { Config.random.nextInt() } - val params = space.materialize() + val params = space.toList() assertEquals(3*100*100, params.size) assertEquals(100*100, params.filter{ it.getString("p1") == "a" @@ -46,8 +46,8 @@ class SearchSpaceTest { @Test fun empty() { val space = EmptySearchSpace() - assertEquals(1, space.size) - assertEquals(1, space.materialize().toList().size) + val params = space.iterator().asSequence().toList() + assertEquals(1, params.size) } diff --git a/roboquant/src/test/kotlin/org/roboquant/loggers/LastEntryLoggerTest.kt b/roboquant/src/test/kotlin/org/roboquant/loggers/LastEntryLoggerTest.kt index 09630a7f5..f4bb8a09e 100644 --- a/roboquant/src/test/kotlin/org/roboquant/loggers/LastEntryLoggerTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/loggers/LastEntryLoggerTest.kt @@ -20,10 +20,7 @@ import org.roboquant.TestData import org.roboquant.common.millis import org.roboquant.common.plus import java.time.Instant -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertTrue +import kotlin.test.* internal class LastEntryLoggerTest { @@ -36,6 +33,7 @@ internal class LastEntryLoggerTest { logger.log(metrics, Instant.now(), "test") logger.end("test") assertTrue(logger.metricNames.isNotEmpty()) + assertContains(logger.metricNames, metrics.keys.first()) val m1 = logger.metricNames.first() val m = logger.getMetric(m1).latestRun()