From 5cc04555cb53f153b4f42071d359a3106c8f7dd2 Mon Sep 17 00:00:00 2001 From: Michael Marte Date: Sun, 10 Dec 2017 18:05:12 +0100 Subject: [PATCH] Added tools for comparing integration test runs --- build.sbt | 1 + doc/copyright | 6 + scripts/compute-speedup.py | 44 ++++++++ scripts/create-results-db.py | 51 +++++++++ scripts/eval-runs.py | 104 ++++++++++++++++++ src/main/yuck/annealing/AnnealingResult.scala | 12 +- src/main/yuck/annealing/RoundLog.scala | 14 +-- .../annealing/StandardAnnealingMonitor.scala | 40 ++++++- src/main/yuck/core/NumericalObjective.scala | 2 +- .../compiler/CompilationContext.scala | 1 + .../flatzinc/compiler/FlatZincCompiler.scala | 8 +- .../compiler/FlatZincCompilerResult.scala | 3 +- .../flatzinc/compiler/ObjectiveFactory.scala | 2 + .../yuck/flatzinc/runner/FlatZincRunner.scala | 2 +- .../runner/FlatZincSolverGenerator.scala | 14 ++- .../flatzinc/test/MiniZincChallenge2012.scala | 4 +- .../flatzinc/test/MiniZincChallenge2013.scala | 2 +- .../flatzinc/test/MiniZincChallenge2015.scala | 8 +- .../flatzinc/test/MiniZincChallenge2017.scala | 2 +- .../yuck/flatzinc/test/MiniZincExamples.scala | 3 +- .../test/util/MiniZincBasedTest.scala | 94 +++++++++++++++- .../flatzinc/test/util/MiniZincTestTask.scala | 1 + 22 files changed, 376 insertions(+), 42 deletions(-) create mode 100755 scripts/compute-speedup.py create mode 100755 scripts/create-results-db.py create mode 100755 scripts/eval-runs.py diff --git a/build.sbt b/build.sbt index 27e62bd4..6bf60187 100644 --- a/build.sbt +++ b/build.sbt @@ -40,6 +40,7 @@ libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test" libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.6" libraryDependencies += "org.jgrapht" % "jgrapht-core" % "1.1.0" libraryDependencies += "com.github.scopt" %% "scopt" % "3.5.0" +libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" // See https://github.com/sbt/junit-interface/issues/66 for why it does not work! testOptions in Test := Seq(Tests.Filter(s => s.endsWith("UnitTestSuite"))) diff --git a/doc/copyright b/doc/copyright index 4b3a9245..af2c7713 100644 --- a/doc/copyright +++ b/doc/copyright @@ -28,5 +28,11 @@ Public License 1.0: * JGraphT (http://www.jgrapht.org) +The following libraries are used under the terms of the Apache +License 2.0: + +* spray-json + (https://github.com/spray/spray-json) + Please see the folder licenses/ for all copyright statements and the licenses. diff --git a/scripts/compute-speedup.py b/scripts/compute-speedup.py new file mode 100755 index 00000000..29c2cf9b --- /dev/null +++ b/scripts/compute-speedup.py @@ -0,0 +1,44 @@ +#! /usr/bin/python3 + +# This script compares the performance of Yuck on two given test runs. +# +# The result database is expected to reside in the working directory under the name results.db. + +import argparse +import json +import numpy +import sqlite3 +import statistics +import sys + +def computeSpeedups(cursor, run1, run2): + tasks = list(cursor.execute('SELECT DISTINCT problem, model, instance, problem_type FROM result ORDER BY problem, model, instance')) + resultQuery = 'SELECT solved, moves_per_second FROM result WHERE run = ? AND problem = ? AND model = ? AND instance = ?' + for problem, model, instance, problemType in tasks: + result1 = cursor.execute(resultQuery, (run1, problem, model, instance)) + solved1, mps1 = result1.fetchone() + result2 = cursor.execute(resultQuery, (run2, problem, model, instance)) + solved2, mps2 = result2.fetchone() + yield {'problem': problem, 'model': model, 'instance': instance, 'speedup': mps2 / mps1 if mps1 and mps2 else None} + +def main(): + parser = argparse.ArgumentParser(description = 'Compares the performance of Yuck on two given test runs') + parser.add_argument('run1', metavar = 'run1') + parser.add_argument('run2', metavar = 'run2') + args = parser.parse_args() + with sqlite3.connect("results.db") as conn: + cursor = conn.cursor() + results = list(computeSpeedups(cursor, args.run1, args.run2)) + speedups = sorted([result['speedup'] for result in results]) + stats = { + 'speedup-min': min(speedups), + 'speedup-max': max(speedups), + 'speedup-mean': statistics.mean(speedups), + # harmonic mean might be more appropriate, but it is available only from Python 3.6 + 'speedup-pstdev': statistics.pstdev(speedups), + 'speedup-median': statistics.median(speedups), + 'speedup-histogram': numpy.histogram(speedups, 'auto')[0].tolist() + } + print(json.dumps(stats, sort_keys = True, indent = 4)) + +main() diff --git a/scripts/create-results-db.py b/scripts/create-results-db.py new file mode 100755 index 00000000..29f8a028 --- /dev/null +++ b/scripts/create-results-db.py @@ -0,0 +1,51 @@ +#! /usr/bin/python3 + +# This script creates a database of Yuck integration test results (results.db). +# +# To import results, call the script with a list of JSON result files on the command line. +# When the database already exists, the given results will be added unless they are +# already in the database. + +import argparse +import json +import sqlite3 +from itertools import repeat + +def createDb(cursor): + cursor.execute('CREATE TABLE IF NOT EXISTS result (run TEXT NOT NULL, suite TEXT NOT NULL, problem TEXT NOT NULL, model TEXT NOT NULL, instance TEXT NOT NULL, problem_type TEXT NOT NULL CONSTRAINT result_problem_type_constraint CHECK (problem_type IN ("MIN", "MAX", "SAT")), optimum INT, high_score INT, solved INT NOT NULL CONSTRAINT result_solved_constraint CHECK (solved in (0, 1)), violation INT CONSTRAINT result_violation_constraint CHECK (violation >= 0), quality INT, runtime_in_seconds DOUBLE CONSTRAINT result_runtime_in_seconds_constraint CHECK (runtime_in_seconds >= 0), moves_per_second DOUBLE CONSTRAINT result_moves_per_second_constraint CHECK (moves_per_second >= 0), CONSTRAINT result_unique_constraint UNIQUE (run, problem, model, instance) ON CONFLICT IGNORE)') + cursor.execute('CREATE INDEX IF NOT EXISTS result_index ON result(run, problem, model, instance)') + +def importResults(run, file, cursor): + results = json.load(file) + solverStatistics = results.get('solver-statistics') + cursor.execute( + 'INSERT INTO result VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + (run, + results['task']['suite'], + results['task']['problem'], + results['task']['model'], + results['task']['instance'], + results['task']['problem-type'], + results['task'].get('optimum'), + results['task'].get('high-score'), + results['result']['solved'], + results['result'].get('violation'), + results['result'].get('quality'), + solverStatistics['runtime-in-seconds'] if solverStatistics else None, + solverStatistics['moves-per-second'] if solverStatistics else None)) + +def main(): + parser = argparse.ArgumentParser(description = 'Puts Yuck integration test results into database') + parser.add_argument('run', metavar = 'run') + parser.add_argument('filenames', metavar = 'json-result-file', nargs = '+') + args = parser.parse_args() + with sqlite3.connect("results.db") as conn: + cursor = conn.cursor() + createDb(cursor) + cursor.execute('PRAGMA foreign_keys = ON'); + for filename in args.filenames: + with open(filename) as file: + importResults(args.run, file, cursor) + conn.commit() + +main() diff --git a/scripts/eval-runs.py b/scripts/eval-runs.py new file mode 100755 index 00000000..88a87e9f --- /dev/null +++ b/scripts/eval-runs.py @@ -0,0 +1,104 @@ +#! /usr/bin/python3 + +# This script helps to evaluate a given set of Yuck integration test runs. +# +# For each instance, the script retrieves the objective value of the best solution +# in order to compute, for each given run, a penalty between 0 and 1 (using feature +# scaling) where 0 means the solution is one of the best and 1 means it is one of the +# worst. (In case there is no solution, the penalty is 1.) +# +# In the end the script prints, for each given run, the number of instances it failed on, +# and the penalties in terms of their mean, standard deviation, and median. +# +# The database is expected to reside in the working directory under the name results.db. +# +# Notice that, by default, feature scaling uses all results that the database provides. +# To restrict the analysis to the given runs, use the -r option. + +import argparse +import json +import numpy +import sqlite3 +import statistics +import sys + +def evalRuns(cursor, args): + jobQuery = 'SELECT DISTINCT problem, model, instance, problem_type FROM result' + if args.problemType: + jobQuery += ' WHERE problem_type = ?' + jobQuery += ' ORDER BY problem, model, instance'; + jobs = list(cursor.execute(jobQuery, (args.problemType, )) if args.problemType else cursor.execute(jobQuery)) + if not jobs: + print('No results found', file = sys.stderr) + return {} + runsInScope = args.runs if args.ignoreOtherRuns else list(map(lambda result: result[0], cursor.execute('SELECT DISTINCT run from result'))); + results = {} + penalties = {} + failures = {} + for run in runsInScope: + resultQuery = 'SELECT solved, quality FROM result WHERE run = ?'; + if args.problemType: + resultQuery += ' AND problem_type = ?' + resultQuery += ' ORDER BY problem, model, instance'; + results[run] = list(cursor.execute(resultQuery, (run, args.problemType)) if args.problemType else cursor.execute(resultQuery, (run,))) + if len(results[run]) != len(jobs): + print('Expected {} results for run {}, but found {}'.format(len(jobs), run, len(results[run])), file = sys.stderr) + return {} + for run in args.runs: + penalties[run] = [] + failures[run] = 0 + for i in range(0, len(jobs)): + (problem, model, instance, problemType) = jobs[i] + qualities = [int(result[1]) for result in [results[run][i] for run in runsInScope] if result[0] and result[1]] + optima = [int(result[0]) for result in cursor.execute('SELECT optimum FROM result WHERE problem = ? AND model = ? AND instance = ? AND optimum IS NOT NULL', (problem, model, instance))] + qualities += optima + highScores = [int(result[0]) for result in cursor.execute('SELECT high_score FROM result WHERE problem = ? AND model = ? AND instance = ? AND high_score IS NOT NULL', (problem, model, instance))] + qualities += highScores + (low, high) = (None, None) if not qualities else (min(qualities), max(qualities)) + if args.verbose: + print('-' * 80) + print(problem, instance, problemType, low, high) + for run in args.runs: + (solved, quality) = results[run][i] + if solved: + if high == low: + penalty = 0 + elif problemType == 'MIN': + penalty = (quality - low) / (high - low) + else: + penalty = 1 - ((quality - low) / (high - low)) + if args.verbose: + print(run, quality, penalty) + else: + failures[run] += 1 + penalty = 1 + penalties[run] += [penalty] + return {run: {'failures': failures[run], 'penalties': penalties[run]} for run in args.runs} + +def postprocessResult(result): + penalties = result['penalties'] + return { + 'failures': result['failures'], + 'penalty-min': min(penalties), + 'penalty-max': max(penalties), + 'penalty-mean': statistics.mean(penalties), + 'penalty-pstdev': statistics.pstdev(penalties), + 'penalty-median': statistics.median(penalties), + 'penalty-histogram': numpy.histogram(penalties, 10, (0, 1))[0].tolist() + } + +def main(): + parser = argparse.ArgumentParser(description = 'Helps to evaluate a given set of Yuck integration test runs') + parser.add_argument('-v', '--verbose', action='store_true') + parser.add_argument('-r', '--ignore-other-runs', dest = 'ignoreOtherRuns', action='store_true', help = 'Ignore results from runs other than the given ones') + parser.add_argument('-t', '--problem-type', dest = 'problemType', choices = ['SAT', 'MIN', 'MAX'], help = 'Restrict analysis to given problem type') + parser.add_argument('runs', metavar = 'run', nargs = '+') + args = parser.parse_args() + with sqlite3.connect("results.db") as conn: + cursor = conn.cursor() + results = evalRuns(cursor, args) + if results: + postprocessedResults = {run: postprocessResult(results[run]) for run in results} + print(json.dumps(postprocessedResults, sort_keys = True, indent = 4)) + +main() diff --git a/src/main/yuck/annealing/AnnealingResult.scala b/src/main/yuck/annealing/AnnealingResult.scala index 9ba6ec51..099f49d6 100644 --- a/src/main/yuck/annealing/AnnealingResult.scala +++ b/src/main/yuck/annealing/AnnealingResult.scala @@ -20,10 +20,10 @@ final class AnnealingResult( var costsOfFinalProposal: Costs = null val roundLogs = new mutable.ArrayBuffer[RoundLog] var indexOfRoundWithBestProposal: Int = 0 - var runtimeInMillis: Long = 0 - def movesPerSecond: Int = roundLogs.toIterator.map(_.movesPerSecond).sum / roundLogs.size - def consultationsPerSecond: Int = roundLogs.toIterator.map(_.consultationsPerSecond).sum / roundLogs.size - def consultationsPerMove: Int = roundLogs.toIterator.map(_.consultationsPerMove).sum / roundLogs.size - def commitmentsPerSecond: Int = roundLogs.toIterator.map(_.commitmentsPerSecond).sum / roundLogs.size - def commitmentsPerMove: Int = roundLogs.toIterator.map(_.commitmentsPerMove).sum / roundLogs.size + def runtimeInSeconds: Double = roundLogs.toIterator.map(_.runtimeInSeconds).sum + def movesPerSecond: Double = roundLogs.toIterator.map(_.movesPerSecond).sum / roundLogs.size + def consultationsPerSecond: Double = roundLogs.toIterator.map(_.consultationsPerSecond).sum / roundLogs.size + def consultationsPerMove: Double = roundLogs.toIterator.map(_.consultationsPerMove).sum / roundLogs.size + def commitmentsPerSecond: Double = roundLogs.toIterator.map(_.commitmentsPerSecond).sum / roundLogs.size + def commitmentsPerMove: Double = roundLogs.toIterator.map(_.commitmentsPerMove).sum / roundLogs.size } diff --git a/src/main/yuck/annealing/RoundLog.scala b/src/main/yuck/annealing/RoundLog.scala index 366a5c10..9a2efc2c 100644 --- a/src/main/yuck/annealing/RoundLog.scala +++ b/src/main/yuck/annealing/RoundLog.scala @@ -8,7 +8,7 @@ import yuck.core._ */ final class RoundLog(val roundIndex: Int) { override def toString = - "%d;%3.6f;%1.6f;%1.6f;%s;%s;%s;%s;%s;%d;%s;%d;%d;%d;%d;%d;%d;%d".format( + "%d;%3.6f;%1.6f;%1.6f;%s;%s;%s;%s;%s;%d;%f;%f;%d;%f;%f;%d;%f;%f".format( roundIndex, temperature, traditionalAcceptanceRatio, uphillAcceptanceRatio, costsOfInitialProposal, costsOfFinalProposal, costsOfBestProposal, @@ -40,12 +40,12 @@ final class RoundLog(val roundIndex: Int) { var numberOfConsultations: Int = 0 /** How often Constraint.commit was called. */ var numberOfCommitments: Int = 0 - def runtimeInSeconds: Double = scala.math.max(1l, runtimeInMillis).toDouble / 1000.0 - def movesPerSecond: Int = (numberOfMonteCarloAttempts.toDouble / runtimeInSeconds).toInt - def consultationsPerSecond: Int = (numberOfConsultations.toDouble / runtimeInSeconds).toInt - def consultationsPerMove: Int = (numberOfConsultations.toDouble / numberOfMonteCarloAttempts.toDouble).toInt - def commitmentsPerSecond: Int = (numberOfCommitments.toDouble / runtimeInSeconds).toInt - def commitmentsPerMove: Int = (numberOfCommitments.toDouble / numberOfMonteCarloAttempts.toDouble).toInt + def runtimeInSeconds: Double = scala.math.max(1l, runtimeInMillis).toDouble / 1000 + def movesPerSecond: Double = numberOfMonteCarloAttempts.toDouble / runtimeInSeconds + def consultationsPerSecond: Double = numberOfConsultations.toDouble / runtimeInSeconds + def consultationsPerMove: Double = numberOfConsultations.toDouble / numberOfMonteCarloAttempts + def commitmentsPerSecond: Double = numberOfCommitments.toDouble / runtimeInSeconds + def commitmentsPerMove: Double = numberOfCommitments.toDouble / numberOfMonteCarloAttempts def updateAcceptanceRatio { val numberOfAcceptedMoves = numberOfMonteCarloAttempts - numberOfRejectedMoves traditionalAcceptanceRatio = numberOfAcceptedMoves.toDouble / numberOfMonteCarloAttempts diff --git a/src/main/yuck/annealing/StandardAnnealingMonitor.scala b/src/main/yuck/annealing/StandardAnnealingMonitor.scala index 025048ea..55190405 100644 --- a/src/main/yuck/annealing/StandardAnnealingMonitor.scala +++ b/src/main/yuck/annealing/StandardAnnealingMonitor.scala @@ -1,5 +1,7 @@ package yuck.annealing +import scala.collection._ + import yuck.core._ import yuck.util.arm.ManagedResource import yuck.util.logging.LazyLogger @@ -52,6 +54,7 @@ class StandardAnnealingMonitor( logger.criticalSection { logger.log("Suspended solver in round %d".format(result.roundLogs.size - 1)) logStatistics(result) + captureSolverStatistics(result) } } } @@ -76,6 +79,7 @@ class StandardAnnealingMonitor( "Solver finished with proposal of quality %s in round %d".format( result.costsOfBestProposal, result.roundLogs.size - 1)) logStatistics(result) + captureSolverStatistics(result) } } @@ -125,13 +129,39 @@ class StandardAnnealingMonitor( logger.withLogScope("Solver statistics".format(result.solverName)) { logger.log("Number of rounds: %d".format(result.roundLogs.size)) if (result.roundLogs.size > 0) { - logger.log("Moves per second: %d".format(result.movesPerSecond)) - logger.log("Consultations per second: %d".format(result.consultationsPerSecond)) - logger.log("Consultations per move: %d".format(result.consultationsPerMove)) - logger.log("Commitments per second: %d".format(result.commitmentsPerSecond)) - logger.log("Commitments per move: %d".format(result.commitmentsPerMove)) + logger.log("Moves per second: %f".format(result.movesPerSecond)) + logger.log("Consultations per second: %f".format(result.consultationsPerSecond)) + logger.log("Consultations per move: %f".format(result.consultationsPerMove)) + logger.log("Commitments per second: %f".format(result.commitmentsPerSecond)) + logger.log("Commitments per move: %f".format(result.commitmentsPerMove)) } } } + private class SolverStatistics( + val runtimeInSeconds: Double, val movesPerSecond: Double, + val consultationsPerSecond: Double, val consultationsPerMove: Double, + val commitmentsPerSecond: Double, val commitmentsPerMove: Double) + + private val solverStatistics = new mutable.ArrayBuffer[SolverStatistics] + + private def captureSolverStatistics(result: AnnealingResult) { + if (! result.roundLogs.isEmpty) { + solverStatistics += + new SolverStatistics( + result.runtimeInSeconds, result.movesPerSecond, + result.consultationsPerSecond, result.consultationsPerMove, + result.commitmentsPerSecond, result.commitmentsPerMove) + } + } + + def wasSearchRequired: Boolean = ! solverStatistics.isEmpty + def numberOfRestarts: Int = scala.math.max(0, solverStatistics.size - 1) + def runtimeInSeconds: Double = solverStatistics.toIterator.map(_.runtimeInSeconds).sum + def movesPerSecond: Double = solverStatistics.toIterator.map(_.movesPerSecond).sum / solverStatistics.size + def consultationsPerSecond: Double = solverStatistics.toIterator.map(_.consultationsPerSecond).sum / solverStatistics.size + def consultationsPerMove: Double = solverStatistics.toIterator.map(_.consultationsPerMove).sum / solverStatistics.size + def commitmentsPerSecond: Double = solverStatistics.toIterator.map(_.commitmentsPerSecond).sum / solverStatistics.size + def commitmentsPerMove: Double = solverStatistics.toIterator.map(_.commitmentsPerMove).sum / solverStatistics.size + } diff --git a/src/main/yuck/core/NumericalObjective.scala b/src/main/yuck/core/NumericalObjective.scala index a8674a6a..fb04bef2 100644 --- a/src/main/yuck/core/NumericalObjective.scala +++ b/src/main/yuck/core/NumericalObjective.scala @@ -9,7 +9,7 @@ import scala.math._ */ abstract class NumericalObjective [Value <: NumericalValue[Value]] - (x: Variable[Value]) + (val x: Variable[Value]) (implicit valueTraits: NumericalValueTraits[Value]) extends AnyObjective { diff --git a/src/main/yuck/flatzinc/compiler/CompilationContext.scala b/src/main/yuck/flatzinc/compiler/CompilationContext.scala index 1b8c44bf..bb07dca4 100644 --- a/src/main/yuck/flatzinc/compiler/CompilationContext.scala +++ b/src/main/yuck/flatzinc/compiler/CompilationContext.scala @@ -29,6 +29,7 @@ final class CompilationContext( val definedVars = new mutable.HashSet[AnyVariable] // variables that occur in a defined_var annotation val costVars = new mutable.ArrayBuffer[Variable[IntegerValue]] var costVar: Variable[IntegerValue] = null + var objectiveVar: Variable[IntegerValue] = null var objective: AnyObjective = null val implicitlyConstrainedVars = new mutable.HashSet[AnyVariable] var maybeNeighbourhood: Option[Neighbourhood] = null diff --git a/src/main/yuck/flatzinc/compiler/FlatZincCompiler.scala b/src/main/yuck/flatzinc/compiler/FlatZincCompiler.scala index 8986e037..f7f17373 100644 --- a/src/main/yuck/flatzinc/compiler/FlatZincCompiler.scala +++ b/src/main/yuck/flatzinc/compiler/FlatZincCompiler.scala @@ -43,13 +43,15 @@ final class FlatZincCompiler logger.criticalSection { logger.withLogScope("Yuck model statistics") { - logModelStatistics(cc) + logYuckModelStatistics(cc) } } val vars = (for ((key, x) <- cc.vars) yield key.toString -> x).toMap val arrays = (for ((key, array) <- cc.arrays) yield key.toString -> array).toMap - new FlatZincCompilerResult(cc.ast, cc.space, vars, arrays, cc.costVar, cc.objective, cc.maybeNeighbourhood) + new FlatZincCompilerResult( + cc.ast, cc.space, vars, arrays, cc.costVar, Option(cc.objectiveVar), cc.objective, + cc.maybeNeighbourhood) } @@ -147,7 +149,7 @@ final class FlatZincCompiler } } - private def logModelStatistics(cc: CompilationContext) = { + private def logYuckModelStatistics(cc: CompilationContext) = { lazy val searchVariables = cc.space.searchVariables logger.logg("Search variables: %s".format(searchVariables)) logger.log("%d search variables".format(searchVariables.size)) diff --git a/src/main/yuck/flatzinc/compiler/FlatZincCompilerResult.scala b/src/main/yuck/flatzinc/compiler/FlatZincCompilerResult.scala index c5e59b98..6bae4ebd 100644 --- a/src/main/yuck/flatzinc/compiler/FlatZincCompilerResult.scala +++ b/src/main/yuck/flatzinc/compiler/FlatZincCompilerResult.scala @@ -14,7 +14,8 @@ final class FlatZincCompilerResult( val space: Space, val vars: immutable.Map[String, AnyVariable], // also holds named parameters val arrays: immutable.Map[String, immutable.IndexedSeq[AnyVariable]], - val costVar: AnyVariable, + val costVar: Variable[IntegerValue], + val maybeObjectiveVar: Option[Variable[IntegerValue]], val objective: AnyObjective, val maybeNeighbourhood: Option[Neighbourhood] ) diff --git a/src/main/yuck/flatzinc/compiler/ObjectiveFactory.scala b/src/main/yuck/flatzinc/compiler/ObjectiveFactory.scala index a92fa993..f470509e 100644 --- a/src/main/yuck/flatzinc/compiler/ObjectiveFactory.scala +++ b/src/main/yuck/flatzinc/compiler/ObjectiveFactory.scala @@ -60,6 +60,7 @@ final class ObjectiveFactory } if (maybeObjectiveVar.isDefined) { objectives += new MinimizationObjective[IntegerValue](maybeObjectiveVar.get, IntegerValue.get(lb), Some(MinusOne)) + cc.objectiveVar = maybeObjectiveVar.get } case Maximize(a, _) => val x = compileExpr[IntegerValue](a) @@ -94,6 +95,7 @@ final class ObjectiveFactory } if (maybeObjectiveVar.isDefined) { objectives += new MaximizationObjective[IntegerValue](maybeObjectiveVar.get, IntegerValue.get(ub), Some(One)) + cc.objectiveVar = maybeObjectiveVar.get } } cc.costVar = createNonNegativeChannel[IntegerValue] diff --git a/src/main/yuck/flatzinc/runner/FlatZincRunner.scala b/src/main/yuck/flatzinc/runner/FlatZincRunner.scala index 678856e3..1764f734 100644 --- a/src/main/yuck/flatzinc/runner/FlatZincRunner.scala +++ b/src/main/yuck/flatzinc/runner/FlatZincRunner.scala @@ -53,7 +53,7 @@ object FlatZincRunner { .action((x, cl) => cl.copy(cfg = cl.cfg.copy(seed = x))) opt[Int]("restart-limit") .text("Default value is %s".format(defaultCfg.restartLimit)) - .action((x, cl) => cl.copy(cfg = cl.cfg.copy(restartLimit = max(1, x)))) + .action((x, cl) => cl.copy(cfg = cl.cfg.copy(restartLimit = max(0, x)))) opt[Int]("target-objective-value") .text("Optional stopping criterion in terms of an objective value") .action((x, cl) => cl.copy(cfg = cl.cfg.copy(maybeTargetObjectiveValue = Some(x)))) diff --git a/src/main/yuck/flatzinc/runner/FlatZincSolverGenerator.scala b/src/main/yuck/flatzinc/runner/FlatZincSolverGenerator.scala index 7b182242..624780a9 100644 --- a/src/main/yuck/flatzinc/runner/FlatZincSolverGenerator.scala +++ b/src/main/yuck/flatzinc/runner/FlatZincSolverGenerator.scala @@ -89,14 +89,11 @@ final class FlatZincSolverGenerator override def call = { logger.withLogScope("FlatZinc model statistics") { - logger.log("%d predicate declarations".format(ast.predDecls.size)) - logger.log("%d parameter declarations".format(ast.paramDecls.size)) - logger.log("%d variable declarations".format(ast.varDecls.size)) - logger.log("%d constraints".format(ast.constraints.size)) + logFlatZincModelStatistics } val randomGenerator = new JavaRandomGenerator(cfg.seed) val solvers = - for (i <- 1 to max(1, cfg.restartLimit)) yield + for (i <- 1 to 1 + max(0, cfg.restartLimit)) yield new OnDemandGeneratedSolver( new BaseSolverGenerator(randomGenerator.nextGen, i), logger, @@ -111,4 +108,11 @@ final class FlatZincSolverGenerator solver } + private def logFlatZincModelStatistics = { + logger.log("%d predicate declarations".format(ast.predDecls.size)) + logger.log("%d parameter declarations".format(ast.paramDecls.size)) + logger.log("%d variable declarations".format(ast.varDecls.size)) + logger.log("%d constraints".format(ast.constraints.size)) + } + } diff --git a/src/test/yuck/flatzinc/test/MiniZincChallenge2012.scala b/src/test/yuck/flatzinc/test/MiniZincChallenge2012.scala index ccb96384..6518e149 100644 --- a/src/test/yuck/flatzinc/test/MiniZincChallenge2012.scala +++ b/src/test/yuck/flatzinc/test/MiniZincChallenge2012.scala @@ -32,7 +32,7 @@ final class MiniZincChallenge2012 extends MiniZincBasedTest { @Test @Category(Array(classOf[MinimizationProblem], classOf[HasCumulativeConstraint], classOf[HasDiffnConstraint])) def carpet_cutting_05 { - solve(task.copy(problemName = "carpet-cutting", modelName = "cc_base", instanceName = "mzn_rnd_test.05")) + solve(task.copy(problemName = "carpet-cutting", modelName = "cc_base", instanceName = "mzn_rnd_test.05", maybeHighScore = Some(1192))) } @Test @@ -130,7 +130,7 @@ final class MiniZincChallenge2012 extends MiniZincBasedTest { @Test @Category(Array(classOf[MinimizationProblem])) def vrp_A_n38_k5 { - solve(task.copy(problemName = "vrp", instanceName = "A-n38-k5.vrp")) + solve(task.copy(problemName = "vrp", instanceName = "A-n38-k5.vrp", maybeHighScore = Some(737))) } } diff --git a/src/test/yuck/flatzinc/test/MiniZincChallenge2013.scala b/src/test/yuck/flatzinc/test/MiniZincChallenge2013.scala index 1ac7ff90..039adf05 100644 --- a/src/test/yuck/flatzinc/test/MiniZincChallenge2013.scala +++ b/src/test/yuck/flatzinc/test/MiniZincChallenge2013.scala @@ -32,7 +32,7 @@ final class MiniZincChallenge2013 extends MiniZincBasedTest { @Test @Category(Array(classOf[MinimizationProblem])) def celar_6_sub2 { - solve(task.copy(problemName = "celar", instanceName = "CELAR6-SUB2")) + solve(task.copy(problemName = "celar", instanceName = "CELAR6-SUB2", maybeHighScore = Some(2746))) } @Test diff --git a/src/test/yuck/flatzinc/test/MiniZincChallenge2015.scala b/src/test/yuck/flatzinc/test/MiniZincChallenge2015.scala index 3761df34..23571992 100644 --- a/src/test/yuck/flatzinc/test/MiniZincChallenge2015.scala +++ b/src/test/yuck/flatzinc/test/MiniZincChallenge2015.scala @@ -26,7 +26,7 @@ final class MiniZincChallenge2015 extends MiniZincBasedTest { @Test @Category(Array(classOf[MinimizationProblem], classOf[HasCircuitConstraint])) def cvrp_simple2 { - solve(task.copy(problemName = "cvrp", instanceName = "simple2")) + solve(task.copy(problemName = "cvrp", instanceName = "simple2", maybeOptimum = Some(34))) } @Test @@ -39,7 +39,7 @@ final class MiniZincChallenge2015 extends MiniZincBasedTest { @Test @Category(Array(classOf[MinimizationProblem], classOf[HasNValueConstraint], classOf[HasAtMostConstraint])) def gfd_schedule_n30f3d30m7k4 { - solve(task.copy(problemName = "gfd-schedule", instanceName = "n30f3d30m7k4")) + solve(task.copy(problemName = "gfd-schedule", instanceName = "n30f3d30m7k4", maybeOptimum = Some(1))) } @Test @@ -57,7 +57,7 @@ final class MiniZincChallenge2015 extends MiniZincBasedTest { @Test @Category(Array(classOf[MinimizationProblem], classOf[HasCumulativeConstraint])) def largescheduling_0100_1 { - solve(task.copy(problemName = "largescheduling", modelName = "largecumulative", instanceName = "instance-0100-1")) + solve(task.copy(problemName = "largescheduling", modelName = "largecumulative", instanceName = "instance-0100-1", maybeHighScore = Some(230502))) } @Test @@ -113,7 +113,7 @@ final class MiniZincChallenge2015 extends MiniZincBasedTest { @Test @Category(Array(classOf[MinimizationProblem], classOf[HasAtLeastConstraint], classOf[HasAtMostConstraint], classOf[HasExactlyConstraint])) def roster_chicroster_dataset_2 { - solve(task.copy(problemName = "roster", modelName = "roster_model", instanceName = "chicroster_dataset_2")) + solve(task.copy(problemName = "roster", modelName = "roster_model", instanceName = "chicroster_dataset_2", maybeOptimum = Some(0))) } @Test diff --git a/src/test/yuck/flatzinc/test/MiniZincChallenge2017.scala b/src/test/yuck/flatzinc/test/MiniZincChallenge2017.scala index f1293785..1ca62b86 100644 --- a/src/test/yuck/flatzinc/test/MiniZincChallenge2017.scala +++ b/src/test/yuck/flatzinc/test/MiniZincChallenge2017.scala @@ -104,7 +104,7 @@ final class MiniZincChallenge2017 extends MiniZincBasedTest { @Test @Category(Array(classOf[MinimizationProblem])) def road_cons_17 { - solve(task.copy(problemName = "road-cons", modelName = "road_naive", instanceName = "road_17", maybeMaximumNumberOfVirtualCores = Some(1))) + solve(task.copy(problemName = "road-cons", modelName = "road_naive", instanceName = "road_17", maybeMaximumNumberOfVirtualCores = Some(1), maybeOptimum = Some(13560))) } @Test diff --git a/src/test/yuck/flatzinc/test/MiniZincExamples.scala b/src/test/yuck/flatzinc/test/MiniZincExamples.scala index 272f58e6..639b0139 100644 --- a/src/test/yuck/flatzinc/test/MiniZincExamples.scala +++ b/src/test/yuck/flatzinc/test/MiniZincExamples.scala @@ -380,8 +380,7 @@ final class HardMiniZincExamples classOf[EasyMiniZincExamples], classOf[MediumMiniZincExamples])) @Test -class TractableMiniZincExamples { -} +class TractableMiniZincExamples /** * @author Michael Marte diff --git a/src/test/yuck/flatzinc/test/util/MiniZincBasedTest.scala b/src/test/yuck/flatzinc/test/util/MiniZincBasedTest.scala index b9ca87b3..b881174d 100644 --- a/src/test/yuck/flatzinc/test/util/MiniZincBasedTest.scala +++ b/src/test/yuck/flatzinc/test/util/MiniZincBasedTest.scala @@ -3,9 +3,11 @@ package yuck.flatzinc.test.util import scala.collection._ import org.junit._ +import spray.json._ import yuck.annealing._ import yuck.core._ +import yuck.flatzinc.ast._ import yuck.flatzinc.compiler.{FlatZincCompilerResult, InconsistentProblemException} import yuck.flatzinc.parser._ import yuck.flatzinc.runner._ @@ -28,12 +30,14 @@ class MiniZincBasedTest extends IntegrationTest { } } + private val jsonNodes = new mutable.ArrayBuffer[JsField] + private def trySolve(task: MiniZincTestTask): Result = { val suitePath = task.suitePath val suiteName = if (task.suiteName.isEmpty) new java.io.File(suitePath).getName else task.suiteName val problemName = task.problemName val modelName = if (task.modelName.isEmpty) problemName else task.modelName - val instanceName = task.instanceName + val instanceName = if (task.instanceName.isEmpty) modelName else task.instanceName val (mznFilePath, dznFilePath, outputDirectoryPath) = task.directoryLayout match { case MiniZincExamplesLayout => ("%s/problems/%s.mzn".format(suitePath, problemName), @@ -83,6 +87,7 @@ class MiniZincBasedTest extends IntegrationTest { maybeTargetObjectiveValue = task.maybeOptimum, maybeQualityTolerance = task.maybeQualityTolerance) val sigint = new SettableSigint + val monitor = new StandardAnnealingMonitor(logger) val result = scoped(new ManagedShutdownHook({logger.log("Received SIGINT"); sigint.set})) { maybeTimeboxed(cfg.maybeRuntimeLimitInSeconds, sigint, "solver", logger) { @@ -90,8 +95,10 @@ class MiniZincBasedTest extends IntegrationTest { logger.withTimedLogScope("Parsing FlatZinc file") { new FlatZincFileParser(fznFilePath, logger).call } - using(new StandardAnnealingMonitor(logger)) { - monitor => new FlatZincSolverGenerator(ast, cfg, sigint, logger, monitor).call.call + logTask(task.copy(suiteName = suiteName, modelName = modelName, instanceName = instanceName), ast) + logFlatZincModelStatistics(ast) + scoped(monitor) { + new FlatZincSolverGenerator(ast, cfg, sigint, logger, monitor).call.call } } } @@ -107,6 +114,14 @@ class MiniZincBasedTest extends IntegrationTest { logger.withLogScope("Best proposal") { new FlatZincResultFormatter(result).call.foreach(logger.log(_)) } + logYuckModelStatistics(result.space) + logResult(result) + logSolverStatistics(monitor) + val jsonDoc = new JsObject(jsonNodes.toMap) + val jsonFilePath = "%s/yuck.json".format(outputDirectoryPath) + val jsonWriter = new java.io.FileWriter(jsonFilePath) + jsonWriter.write(jsonDoc.prettyPrint) + jsonWriter.close Assert.assertTrue( "No solution found, quality of best proposal was %s".format(result.costsOfBestProposal), result.isSolution) @@ -118,6 +133,79 @@ class MiniZincBasedTest extends IntegrationTest { result } + private def logTask(task: MiniZincTestTask, ast: FlatZincAst) { + val problemType = + ast.solveGoal match { + case Satisfy(_) => "SAT" + case Minimize(_, _) => "MIN" + case Maximize(_, _) => "MAX" + } + val taskNodes = new mutable.ArrayBuffer[JsField] + taskNodes ++= List( + "suite" -> JsString(task.suiteName), + "problem" -> JsString(task.problemName), + "model" -> JsString(task.modelName), + "instance" -> JsString(task.instanceName), + "problem-type" -> JsString(problemType) + ) + if (task.maybeOptimum.isDefined) { + taskNodes += "optimum" -> JsNumber(task.maybeOptimum.get) + } + if (task.maybeHighScore.isDefined) { + taskNodes += "high-score" -> JsNumber(task.maybeHighScore.get) + } + jsonNodes += "task" -> JsObject(taskNodes.toMap) + } + + private def logFlatZincModelStatistics(ast: FlatZincAst) { + jsonNodes += + "flatzinc-model-statistics" -> JsObject( + "number-of-predicate-declarations" -> JsNumber(ast.predDecls.size), + "number-of-parameter-declarations" -> JsNumber(ast.paramDecls.size), + "number-of-variable-declarations" -> JsNumber(ast.varDecls.size), + "number-of-constraints" -> JsNumber(ast.constraints.size) + ) + } + + private def logYuckModelStatistics(space: Space) { + jsonNodes += + "yuck-model-statistics" -> JsObject( + "number-of-search-variables" -> JsNumber(space.searchVariables.size), + "number-of-channel-variables" -> JsNumber(space.channelVariables.size), + // dangling variables are not readily available + "number-of-constraints" -> JsNumber(space.numberOfConstraints), + "number-of-implicit-constraints" -> JsNumber(space.numberOfImplicitConstraints) + ) + } + + private def logResult(result: Result) { + val compilerResult = result.maybeUserData.get.asInstanceOf[FlatZincCompilerResult] + val resultNodes = new mutable.ArrayBuffer[JsField] + resultNodes += "solved" -> JsBoolean(result.isSolution) + if (! result.isSolution) { + resultNodes += "violation" -> JsNumber(result.bestProposal.value(compilerResult.costVar).value) + } + if (compilerResult.maybeObjectiveVar.isDefined) { + resultNodes += "quality" -> JsNumber(result.bestProposal.value(compilerResult.maybeObjectiveVar.get).value) + } + jsonNodes += "result" -> JsObject(resultNodes.toMap) + } + + private def logSolverStatistics(monitor: StandardAnnealingMonitor) { + if (monitor.wasSearchRequired) { + jsonNodes += + "solver-statistics" -> JsObject( + "number-of-restarts" -> JsNumber(monitor.numberOfRestarts), + "runtime-in-seconds" -> JsNumber(monitor.runtimeInSeconds), + "moves-per-second" -> JsNumber(monitor.movesPerSecond), + "consultations-per-second" -> JsNumber(monitor.consultationsPerSecond), + "consultations-per-move" -> JsNumber(monitor.consultationsPerMove), + "commitments-per-second" -> JsNumber(monitor.commitmentsPerSecond), + "commitments-per-move" -> JsNumber(monitor.commitmentsPerMove) + ) + } + } + private def logViolatedConstraints(result: Result) { val visited = new mutable.HashSet[AnyVariable] val compilerResult = result.maybeUserData.get.asInstanceOf[FlatZincCompilerResult] diff --git a/src/test/yuck/flatzinc/test/util/MiniZincTestTask.scala b/src/test/yuck/flatzinc/test/util/MiniZincTestTask.scala index 91e21acb..d543c104 100644 --- a/src/test/yuck/flatzinc/test/util/MiniZincTestTask.scala +++ b/src/test/yuck/flatzinc/test/util/MiniZincTestTask.scala @@ -31,6 +31,7 @@ case class MiniZincTestTask( val maybeRoundLimit: Option[Int] = None, // overrules solverConfiguration.maybeRoundLimitInSeconds val maybeRuntimeLimitInSeconds: Option[Int] = Some(300), // overrules solverConfiguration.maybeRuntimeLimitInSeconds val maybeOptimum: Option[Int] = None, // overrules solverConfiguration.maybeTargetObjectiveValue + val maybeHighScore: Option[Int] = None, // best ever recorded objective value val maybeQualityTolerance: Option[Int] = None, // overrules solverConfiguration.maybeQualityTolerance val logLevel: yuck.util.logging.LogLevel = yuck.util.logging.InfoLogLevel) {