Include ExecutionResult in ScoringResult (#106)

We need this as a preparation for custom scorer support. - Make scorers output debug messages (e.g. diff) to stderr - Integration tests for DiffScorer instead of just unit tests (actually run the diff command in a shell)
ia-toki · Dec 13, 2016 · b3fa95a · b3fa95a
1 parent a5cc921
commit b3fa95a
Show file tree

Hide file tree

Showing 16 changed files with 144 additions and 105 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -10,7 +10,7 @@ install:
   - sudo update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-4.8 90
 
 script:
-  - cmake . && cmake --build . && ./tests && ./ete_tests
+  - cmake . && cmake --build . && ./tests && ./integration_tests && ./ete_tests
 
 after_success:
   - bash <(curl -s https://codecov.io/bash)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -163,7 +163,6 @@ set(SOURCES
         test/tcframe/runner/RunnerTests.cpp
         test/tcframe/runner/RunnerLoggerTests.cpp
         test/tcframe/runner/SlugParserTests.cpp
-        test/tcframe/scorer/DiffScorerTests.cpp
         test/tcframe/scorer/MockScorer.hpp
         test/tcframe/spec/constraint/ConstraintSuiteBuilderTests.cpp
         test/tcframe/spec/core/BaseTestSpecTests.cpp
@@ -207,6 +206,25 @@ target_link_libraries(tests
         gcov
 )
 
+set(INTEGRATION_SOURCES
+        integration-test/tcframe/scorer/DiffScorerIntegrationTests.cpp
+)
+
+add_executable(integration_tests ${INTEGRATION_SOURCES})
+
+target_link_libraries(integration_tests
+        gtest
+        gmock
+        ${CMAKE_THREAD_LIBS_INIT}
+        gcov
+)
+
+add_custom_command(
+        TARGET integration_tests
+        POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/integration-test/resources $<TARGET_FILE_DIR:integration_tests>/integration
+)
+
 set(ETE_SOURCES
         ete-test/tcframe/BaseEteTests.cpp
         ete-test/tcframe/GenerationEteTests.cpp

diff --git a/include/tcframe/generator/TestCaseGenerator.hpp b/include/tcframe/generator/TestCaseGenerator.hpp
@@ -14,6 +14,7 @@
 #include "tcframe/os.hpp"
 #include "tcframe/scorer.hpp"
 #include "tcframe/spec.hpp"
+#include "tcframe/util.hpp"
 #include "tcframe/verdict.hpp"
 #include "tcframe/verifier.hpp"
 
@@ -190,7 +191,8 @@ class TestCaseGenerator {
 
         ScoringResult scoringResult = scorer_->score(inputFilename, outputFilename, "_evaluation.out");
         if (!(scoringResult.verdict() == Verdict::ac())) {
-            throw GenerationException([=] { logger_->logSampleTestCaseCheckFailure(scoringResult.message()); });
+            throw GenerationException([=] { logger_->logSampleTestCaseCheckFailure(
+                    StringUtils::streamToString(scoringResult.executionResult().errorStream())); });
         }
     }
 };

diff --git a/include/tcframe/grader/TestCaseGrader.hpp b/include/tcframe/grader/TestCaseGrader.hpp
@@ -6,6 +6,7 @@
 #include "GraderLogger.hpp"
 #include "tcframe/evaluator.hpp"
 #include "tcframe/scorer.hpp"
+#include "tcframe/util.hpp"
 
 using std::string;
 
@@ -66,7 +67,7 @@ class TestCaseGrader {
 
         logger_->logTestCaseVerdict(result.verdict());
         if (result.verdict() == Verdict::wa()) {
-            logger_->logTestCaseScoringMessage(result.message());
+            logger_->logTestCaseScoringMessage(StringUtils::streamToString(result.executionResult().errorStream()));
         }
         return result.verdict();
     }

diff --git a/include/tcframe/logger/BaseLogger.hpp b/include/tcframe/logger/BaseLogger.hpp
@@ -1,13 +1,9 @@
 #pragma once
 
-#include <streambuf>
-
 #include "LoggerEngine.hpp"
 #include "tcframe/os.hpp"
 #include "tcframe/util.hpp"
 
-using std::istreambuf_iterator;
-
 namespace tcframe {
 
 class BaseLogger {
@@ -39,7 +35,7 @@ class BaseLogger {
         if (result.info().exitCode()) {
             engine_->logListItem2(3, "Exit code: " + StringUtils::toString(result.info().exitCode().value()));
             engine_->logListItem2(3, "Standard error: " + string(
-                    istreambuf_iterator<char>(*result.errorStream()), istreambuf_iterator<char>()));
+                    StringUtils::streamToString(result.errorStream())));
         } else {
             engine_->logListItem2(3, "Exit signal: " + result.info().exitSignal().value());
         }

diff --git a/include/tcframe/scorer/DiffScorer.hpp b/include/tcframe/scorer/DiffScorer.hpp
@@ -1,14 +1,12 @@
 #pragma once
 
-#include <streambuf>
 #include <string>
 
 #include "Scorer.hpp"
 #include "ScoringResult.hpp"
 #include "tcframe/os.hpp"
 #include "tcframe/verdict.hpp"
 
-using std::istreambuf_iterator;
 using std::string;
 
 namespace tcframe {
@@ -23,37 +21,32 @@ class DiffScorer : public Scorer {
     DiffScorer(OperatingSystem* os)
             : os_(os) {}
 
-    ScoringResult score(
-            const string&,
-            const string& outputFilename,
-            const string& evaluationFilename) {
-
-        string briefDiffCommand = "diff --brief " + outputFilename + " " + evaluationFilename;
-        ExecutionResult briefResult = os_->execute(ExecutionRequestBuilder().setCommand(briefDiffCommand).build());
-
-        if (briefResult.info().isSuccessful()) {
-            return ScoringResultBuilder()
-                    .setVerdict(Verdict::ac())
-                    .build();
-        }
-
-        string diffCommand = string() +
-                 "diff " +
-                 "--unchanged-line-format=' %.2dn    %L' " +
-                 "--old-line-format='(expected) [line %.2dn]    %L' " +
-                 "--new-line-format='(received) [line %.2dn]    %L' " +
-                 outputFilename + " " + evaluationFilename;
-        string scoringCommand = diffCommand + " | head -n 10";
+    ScoringResult score(const string&, const string& outputFilename, const string& evaluationFilename) {
+        string scoringCommand = string()
+                + "diff --brief " + outputFilename + " " + evaluationFilename + " > /dev/null"
+                + " || "
+                + "( "
+                + "echo 'Diff:' 1>&2; "
+                + "diff "
+                + "--unchanged-line-format=' %.2dn    %L' "
+                + "--old-line-format='(expected) [line %.2dn]    %L' "
+                + "--new-line-format='(received) [line %.2dn]    %L' "
+                + outputFilename + " " + evaluationFilename
+                + " | head -n 10 1>&2; "
+                + "exit 1;"
+                + " )";
+
         ExecutionResult result = os_->execute(ExecutionRequestBuilder()
                 .setCommand(scoringCommand)
-                .setOutputFilename("_diff.out")
+                .setErrorFilename("_diff.out")
                 .build());
-
-        string diff = string(istreambuf_iterator<char>(*result.outputStream()), istreambuf_iterator<char>());
+        Verdict verdict = result.info().isSuccessful()
+                          ? Verdict::ac()
+                          : Verdict::wa();
 
         return ScoringResultBuilder()
-                .setVerdict(Verdict::wa())
-                .setMessage("Diff:\n" + diff)
+                .setExecutionResult(result)
+                .setVerdict(verdict)
                 .build();
     }
 };

diff --git a/include/tcframe/scorer/ScoringResult.hpp b/include/tcframe/scorer/ScoringResult.hpp
@@ -4,6 +4,7 @@
 #include <tuple>
 #include <utility>
 
+#include "tcframe/os.hpp"
 #include "tcframe/verdict.hpp"
 
 using std::move;
@@ -16,20 +17,20 @@ struct ScoringResult {
     friend class ScoringResultBuilder;
 
 private:
+    ExecutionResult executionResult_;
     Verdict verdict_;
-    string message_;
 
 public:
-    const Verdict& verdict() const {
-        return verdict_;
+    const ExecutionResult& executionResult() const {
+        return executionResult_;
     }
 
-    const string& message() const {
-        return message_;
+    const Verdict& verdict() const {
+        return verdict_;
     }
 
     bool operator==(const ScoringResult& o) const {
-        return tie(verdict_, message_) == tie(o.verdict_, o.message_);
+        return tie(executionResult_, verdict_) == tie(o.executionResult_, o.verdict_);
     }
 };
 
@@ -38,13 +39,13 @@ class ScoringResultBuilder {
     ScoringResult subject_;
 
 public:
-    ScoringResultBuilder& setVerdict(Verdict verdict) {
-        subject_.verdict_ = verdict;
+    ScoringResultBuilder& setExecutionResult(ExecutionResult executionResult) {
+        subject_.executionResult_ = executionResult;
         return *this;
     }
 
-    ScoringResultBuilder& setMessage(string message) {
-        subject_.message_ = message;
+    ScoringResultBuilder& setVerdict(Verdict verdict) {
+        subject_.verdict_ = verdict;
         return *this;
     }
 

diff --git a/include/tcframe/util/StringUtils.hpp b/include/tcframe/util/StringUtils.hpp
@@ -1,11 +1,15 @@
 #pragma once
 
+#include <istream>
 #include <sstream>
+#include <streambuf>
 #include <string>
 #include <vector>
 
 #include "optional.hpp"
 
+using std::istream;
+using std::istreambuf_iterator;
 using std::istringstream;
 using std::ostringstream;
 using std::string;
@@ -24,6 +28,10 @@ class StringUtils {
         return out.str();
     }
 
+    static string streamToString(istream* in) {
+        return string(istreambuf_iterator<char>(*in), istreambuf_iterator<char>());
+    }
+
     template<typename T>
     static optional<T> toNumber(const string& s) {
         istringstream in(s);

diff --git a/integration-test/resources/scorer/diff/contestant_ac.out b/integration-test/resources/scorer/diff/contestant_ac.out
@@ -0,0 +1,2 @@
+yes
+1 2 3
diff --git a/integration-test/resources/scorer/diff/contestant_wa.out b/integration-test/resources/scorer/diff/contestant_wa.out
@@ -0,0 +1,2 @@
+yes
+1 4 5
diff --git a/integration-test/resources/scorer/judge.out b/integration-test/resources/scorer/judge.out
@@ -0,0 +1,2 @@
+yes
+1 2 3
diff --git a/integration-test/tcframe/scorer/DiffScorerIntegrationTests.cpp b/integration-test/tcframe/scorer/DiffScorerIntegrationTests.cpp
@@ -0,0 +1,39 @@
+#include "gmock/gmock.h"
+
+#include "tcframe/scorer/DiffScorer.hpp"
+#include "tcframe/util.hpp"
+
+using ::testing::Eq;
+using ::testing::HasSubstr;
+using ::testing::StartsWith;
+using ::testing::Test;
+
+namespace tcframe {
+
+class DiffScorerIntegrationTests : public Test {
+protected:
+    DiffScorer scorer = DiffScorer(new UnixOperatingSystem());
+};
+
+TEST_F(DiffScorerIntegrationTests, Scoring_AC) {
+    ScoringResult result = scorer.score(
+            "",
+            "integration/scorer/judge.out",
+            "integration/scorer/diff/contestant_ac.out");
+    EXPECT_THAT(result.verdict(), Eq(Verdict::ac()));
+}
+
+TEST_F(DiffScorerIntegrationTests, Scoring_WA) {
+    ScoringResult result = scorer.score(
+            "",
+            "integration/scorer/judge.out",
+            "integration/scorer/diff/contestant_wa.out");
+    EXPECT_THAT(result.verdict(), Eq(Verdict::wa()));
+
+    string scorerMessage = StringUtils::streamToString(result.executionResult().errorStream());
+    EXPECT_THAT(scorerMessage, StartsWith("Diff:\n"));
+    EXPECT_THAT(scorerMessage, HasSubstr("(expected) [line 02]    1 2 3\n"));
+    EXPECT_THAT(scorerMessage, HasSubstr("(received) [line 02]    1 4 5\n"));
+}
+
+}
diff --git a/test/tcframe/generator/TestCaseGeneratorTests.cpp b/test/tcframe/generator/TestCaseGeneratorTests.cpp
@@ -28,6 +28,7 @@ using ::testing::Throw;
 using ::testing::Truly;
 using ::testing::WithArg;
 
+using std::istringstream;
 using std::ostringstream;
 
 namespace tcframe {
@@ -126,7 +127,12 @@ class TestCaseGeneratorTests : public Test {
         ON_CALL(scorer, score(_, _, _))
                 .WillByDefault(DoAll(
                         WithArg<2>(Invoke(&evaluationCaptor2, &Captor<string>::capture)),
-                        InvokeWithoutArgs([] {return ScoringResultBuilder().setVerdict(Verdict::ac()).build();})));
+                        InvokeWithoutArgs([] {return ScoringResultBuilder()
+                                .setExecutionResult(ExecutionResultBuilder()
+                                        .setInfo(ExecutionInfoBuilder().setExitCode(0).build())
+                                        .build())
+                                .setVerdict(Verdict::ac())
+                                .build();})));
     }
 
     struct InputStreamContentIs {
@@ -226,8 +232,11 @@ TEST_F(TestCaseGeneratorTests, Generation_Sample_WithOutput_Successful) {
 TEST_F(TestCaseGeneratorTests, Generation_Sample_WithOutput_Failed_Check) {
     ON_CALL(scorer, score(_, _, _))
             .WillByDefault(Return(ScoringResultBuilder()
+                    .setExecutionResult(ExecutionResultBuilder()
+                            .setInfo(ExecutionInfoBuilder().setExitCode(0).build())
+                            .setErrorStream(new istringstream("diff"))
+                            .build())
                     .setVerdict(Verdict::wa())
-                    .setMessage("diff")
                     .build()));
     {
         InSequence sequence;

diff --git a/test/tcframe/grader/TestCaseGraderTests.cpp b/test/tcframe/grader/TestCaseGraderTests.cpp
@@ -2,6 +2,8 @@
 #include "../helper.hpp"
 #include "../mock.hpp"
 
+#include <sstream>
+
 #include "../evaluator/MockEvaluator.hpp"
 #include "../scorer/MockScorer.hpp"
 #include "MockGraderLogger.hpp"
@@ -17,6 +19,8 @@ using ::testing::Return;
 using ::testing::Test;
 using ::testing::WithArg;
 
+using std::istringstream;
+
 namespace tcframe {
 
 class TestCaseGraderTests : public Test {
@@ -52,7 +56,12 @@ class TestCaseGraderTests : public Test {
         ON_CALL(scorer, score(_, _, _))
                 .WillByDefault(DoAll(
                         WithArg<2>(Invoke(&evaluationCaptor2, &Captor<string>::capture)),
-                        InvokeWithoutArgs([] {return ScoringResultBuilder().setVerdict(Verdict::ac()).build();})));
+                        InvokeWithoutArgs([] {return ScoringResultBuilder()
+                                .setExecutionResult(ExecutionResultBuilder()
+                                        .setInfo(ExecutionInfoBuilder().setExitCode(0).build())
+                                        .build())
+                                .setVerdict(Verdict::ac())
+                                .build();})));
     }
 };
 
@@ -72,8 +81,11 @@ TEST_F(TestCaseGraderTests, Grading_AC) {
 TEST_F(TestCaseGraderTests, Grading_WA) {
     ON_CALL(scorer, score(_, _, _))
             .WillByDefault(Return(ScoringResultBuilder()
+                    .setExecutionResult(ExecutionResultBuilder()
+                            .setInfo(ExecutionInfoBuilder().setExitCode(0).build())
+                            .setErrorStream(new istringstream("diff"))
+                            .build())
                     .setVerdict(Verdict::wa())
-                    .setMessage("diff")
                     .build()));
     {
         InSequence sequence;