Disallow literal sample output if NoOutput() is set (#105)

ia-toki · Dec 13, 2016 · a5cc921 · a5cc921
1 parent b37a8d5
commit a5cc921
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 16 deletions.
diff --git a/include/tcframe/generator/GeneratorLogger.hpp b/include/tcframe/generator/GeneratorLogger.hpp
@@ -95,6 +95,10 @@ class GeneratorLogger : public BaseLogger {
         engine_->logListItem2(3, scoringMessage);
     }
 
+    virtual void logSampleTestCaseNoOutputNeededFailure() {
+        engine_->logListItem1(2, "Problem does not need test case outputs, but this sample test case has output");
+    }
+
     virtual void logSimpleFailure(const string& message) {
         engine_->logListItem1(2, message);
     }

diff --git a/include/tcframe/generator/TestCaseGenerator.hpp b/include/tcframe/generator/TestCaseGenerator.hpp
@@ -61,9 +61,7 @@ class TestCaseGenerator {
             applyInput(testCase);
             verifyInput(testCase);
             generateInput(testCase, inputFilename, config);
-            if (config.needsOutput()) {
-                evaluateAndApplyOutput(testCase, inputFilename, outputFilename, config);
-            }
+            evaluateAndApplyOutput(testCase, inputFilename, outputFilename, config);
         } catch (GenerationException& e) {
             logger_->logTestCaseFailedResult(testCase.description());
             e.callback()();
@@ -116,6 +114,14 @@ class TestCaseGenerator {
             const string& outputFilename,
             const GeneratorConfig& config) {
 
+        optional<string> maybeSampleOutputString = getSampleOutputString(testCase);
+        if (!config.needsOutput()) {
+            if (maybeSampleOutputString) {
+                throw GenerationException([=] { logger_->logSampleTestCaseNoOutputNeededFailure(); });
+            }
+            return;
+        }
+
         EvaluatorConfig evaluatorConfig = EvaluatorConfigBuilder()
                 .setSolutionCommand(config.solutionCommand())
                 .build();
@@ -126,13 +132,22 @@ class TestCaseGenerator {
             throw GenerationException([=] { logger_->logSolutionExecutionFailure(executionResult); });
         }
 
-        checkSampleOutput(testCase, inputFilename, outputFilename, config);
+        if (maybeSampleOutputString) {
+            checkSampleOutput(maybeSampleOutputString.value(), inputFilename, outputFilename, config);
+        }
 
         istream* output = executionResult.outputStream();
         modifyOutputForMultipleTestCases(output, config);
         ioManipulator_->parseOutput(output);
     }
 
+    optional<string> getSampleOutputString(const TestCase& testCase) {
+        if (testCase.data()->type() != TestCaseDataType::SAMPLE) {
+            return optional<string>();
+        }
+        return ((SampleTestCaseData*) testCase.data())->output();
+    }
+
     void modifyInputForMultipleTestCases(ostream* input, const GeneratorConfig& config) {
         if (config.multipleTestCasesCounter() != nullptr) {
             int testCaseNo = 1;
@@ -161,24 +176,16 @@ class TestCaseGenerator {
     }
 
     void checkSampleOutput(
-            const TestCase& testCase,
+            const string& sampleOutputString,
             const string& inputFilename,
             const string& outputFilename,
             const GeneratorConfig& config) {
 
-        if (testCase.data()->type() != TestCaseDataType::SAMPLE) {
-            return;
-        }
-        optional<string> maybeSampleOutput = ((SampleTestCaseData*) testCase.data())->output();
-        if (!maybeSampleOutput) {
-            return;
-        }
-
-        string sampleOutputString = maybeSampleOutput.value();
-        modifySampleOutputStringForMultipleTestCases(sampleOutputString, config);
+        string modifiedSampleOutputString = sampleOutputString;
+        modifySampleOutputStringForMultipleTestCases(modifiedSampleOutputString, config);
 
         ostream* sampleOutput = os_->openForWriting("_evaluation.out");
-        *sampleOutput << sampleOutputString;
+        *sampleOutput << modifiedSampleOutputString;
         os_->closeOpenedWritingStream(sampleOutput);
 
         ScoringResult scoringResult = scorer_->score(inputFilename, outputFilename, "_evaluation.out");

diff --git a/test/tcframe/generator/GeneratorLoggerTests.cpp b/test/tcframe/generator/GeneratorLoggerTests.cpp
@@ -86,6 +86,12 @@ TEST_F(GeneratorLoggerTests, SampleTestCaseCheckFailure) {
     logger.logSampleTestCaseCheckFailure("diff");
 }
 
+TEST_F(GeneratorLoggerTests, SampleTestCaseNoOutputNeededFailure) {
+    EXPECT_CALL(engine,
+                logListItem1(2, "Problem does not need test case outputs, but this sample test case has output"));
+    logger.logSampleTestCaseNoOutputNeededFailure();
+}
+
 TEST_F(GeneratorLoggerTests, ConstraintsVerificationFailure_WithSubtasks) {
     {
         InSequence sequence;

diff --git a/test/tcframe/generator/MockGeneratorLogger.hpp b/test/tcframe/generator/MockGeneratorLogger.hpp
@@ -27,6 +27,7 @@ class MockGeneratorLogger : public GeneratorLogger {
     MOCK_METHOD1(logMultipleTestCasesConstraintsVerificationFailure, void(
             const MultipleTestCasesConstraintsVerificationResult&));
     MOCK_METHOD1(logSampleTestCaseCheckFailure, void(const string&));
+    MOCK_METHOD0(logSampleTestCaseNoOutputNeededFailure, void());
     MOCK_METHOD1(logSimpleFailure, void(const string&));
 };
 

diff --git a/test/tcframe/generator/TestCaseGeneratorTests.cpp b/test/tcframe/generator/TestCaseGeneratorTests.cpp
@@ -238,6 +238,16 @@ TEST_F(TestCaseGeneratorTests, Generation_Sample_WithOutput_Failed_Check) {
     EXPECT_FALSE(generator.generate(sampleTestCaseWithOutput, config));
 }
 
+TEST_F(TestCaseGeneratorTests, Generation_Sample_WithOutput_Failed_NoOutputNeeded) {
+    {
+        InSequence sequence;
+        EXPECT_CALL(logger, logTestCaseFailedResult(optional<string>()));
+        EXPECT_CALL(logger, logSampleTestCaseNoOutputNeededFailure());
+    }
+
+    EXPECT_FALSE(generator.generate(sampleTestCaseWithOutput, noOutputConfig));
+}
+
 TEST_F(TestCaseGeneratorTests, Generation_Successful) {
     {
         InSequence sequence;