diff --git a/optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/config/SolverBenchmarkConfig.java b/optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/config/SolverBenchmarkConfig.java
index 746affdc9c..33c0eac434 100644
--- a/optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/config/SolverBenchmarkConfig.java
+++ b/optaplanner-benchmark/src/main/java/org/optaplanner/benchmark/config/SolverBenchmarkConfig.java
@@ -25,6 +25,7 @@
 import org.optaplanner.core.config.SolverConfigContext;
 import org.optaplanner.core.config.solver.SolverConfig;
 import org.optaplanner.core.config.util.ConfigUtils;
+import org.optaplanner.core.impl.domain.solution.descriptor.SolutionDescriptor;
 
 @XStreamAlias("solverBenchmark")
 public class SolverBenchmarkConfig<Solution_> extends AbstractConfig<SolverBenchmarkConfig> {
@@ -86,8 +87,17 @@ public void buildSolverBenchmark(SolverConfigContext solverConfigContext, Planne
         solverBenchmarkResult.setName(name);
         solverBenchmarkResult.setSubSingleCount(ConfigUtils.inheritOverwritableProperty(subSingleCount, 1));
         solverBenchmarkResult.setSolverConfig(solverConfig);
+        SolutionDescriptor<Object> solutionDescriptor = solverConfig.buildSolutionDescriptor(solverConfigContext);
+        for (Solution_ extraProblem : extraProblems) {
+            if (!solutionDescriptor.getSolutionClass().isInstance(extraProblem)) {
+                throw new IllegalArgumentException("The solverBenchmark name (" + name
+                        + ") for solution class (" + solutionDescriptor.getSolutionClass()
+                        + ") cannot solve a problem (" + extraProblem
+                        + ") of class (" + (extraProblem == null ? null : extraProblem.getClass()) + ").");
+            }
+        }
         solverBenchmarkResult.setScoreDefinition(
-                solverConfig.buildSolutionDescriptor(solverConfigContext).getScoreDefinition());
+                solutionDescriptor.getScoreDefinition());
         solverBenchmarkResult.setSingleBenchmarkResultList(new ArrayList<>());
         ProblemBenchmarksConfig problemBenchmarksConfig_
                 = problemBenchmarksConfig == null ? new ProblemBenchmarksConfig()
diff --git a/optaplanner-benchmark/src/test/java/org/optaplanner/benchmark/api/PlannerBenchmarkFactoryTest.java b/optaplanner-benchmark/src/test/java/org/optaplanner/benchmark/api/PlannerBenchmarkFactoryTest.java
index 775be7fb45..02c50717fc 100644
--- a/optaplanner-benchmark/src/test/java/org/optaplanner/benchmark/api/PlannerBenchmarkFactoryTest.java
+++ b/optaplanner-benchmark/src/test/java/org/optaplanner/benchmark/api/PlannerBenchmarkFactoryTest.java
@@ -18,14 +18,21 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.optaplanner.benchmark.config.SolverBenchmarkConfig;
 import org.optaplanner.core.api.solver.DivertingClassLoader;
+import org.optaplanner.core.api.solver.SolverFactory;
 import org.optaplanner.core.config.phase.custom.CustomPhaseConfig;
 import org.optaplanner.core.impl.phase.custom.NoChangeCustomPhaseCommand;
+import org.optaplanner.core.impl.testdata.domain.TestdataEntity;
+import org.optaplanner.core.impl.testdata.domain.TestdataSolution;
+import org.optaplanner.core.impl.testdata.domain.TestdataValue;
+import org.optaplanner.core.impl.testdata.domain.customcloner.TestdataCorrectlyClonedSolution;
+import org.optaplanner.core.impl.testdata.util.PlannerTestUtils;
 
 import static org.junit.Assert.*;
 
@@ -39,6 +46,40 @@ public static void setup() throws IOException {
         new File(benchmarkTestDir, "output/").mkdir();
     }
 
+    @Test
+    public void createFromSolverFactory() {
+        SolverFactory<TestdataSolution> solverFactory = PlannerTestUtils.buildSolverFactory(
+                TestdataSolution.class, TestdataEntity.class);
+        PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverFactory(
+                solverFactory);
+        TestdataSolution solution = new TestdataSolution("s1");
+        solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3")));
+        solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2")));
+        PlannerBenchmark benchmark = benchmarkFactory.buildPlannerBenchmark(solution);
+        benchmark.benchmark();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void problemIsNotASolutionInstance() {
+        SolverFactory<TestdataSolution> solverFactory = PlannerTestUtils.buildSolverFactory(
+                TestdataSolution.class, TestdataEntity.class);
+        PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverFactory(
+                solverFactory);
+        benchmarkFactory.buildPlannerBenchmark("This is not a solution instance.");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void problemIsNull() {
+        SolverFactory<TestdataSolution> solverFactory = PlannerTestUtils.buildSolverFactory(
+                TestdataSolution.class, TestdataEntity.class);
+        PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverFactory(
+                solverFactory);
+        TestdataSolution solution = new TestdataSolution("s1");
+        solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3")));
+        solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2")));
+        benchmarkFactory.buildPlannerBenchmark(solution, null);
+    }
+
     @Test
     public void benchmarkConfig() {
         PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource(