Support --brief output in local grading (#140)

Resolve #139
ia-toki · May 29, 2017 · 2897c4e · 2897c4e
1 parent 69724ca
commit 2897c4e
Show file tree

Hide file tree

Showing 20 changed files with 262 additions and 27 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -73,9 +73,11 @@ set(INCLUDE
         include/tcframe/generator/GeneratorLogger.hpp
         include/tcframe/generator/TestCaseGenerator.hpp
         include/tcframe/grader.hpp
+        include/tcframe/grader/BriefGraderLogger.hpp
         include/tcframe/grader/DefaultGraderLogger.hpp
         include/tcframe/grader/Grader.hpp
         include/tcframe/grader/GraderLogger.hpp
+        include/tcframe/grader/GraderLoggerFactory.hpp
         include/tcframe/grader/GradingOptions.hpp
         include/tcframe/grader/TestCaseGrader.hpp
         include/tcframe/io_manipulator.hpp
@@ -174,10 +176,12 @@ set(TEST_UNIT
         test/unit/tcframe/generator/MockGeneratorLogger.hpp
         test/unit/tcframe/generator/MockTestCaseGenerator.hpp
         test/unit/tcframe/generator/TestCaseGeneratorTests.cpp
+        test/unit/tcframe/grader/BriefGraderLoggerTests.cpp
         test/unit/tcframe/grader/DefaultGraderLoggerTests.cpp
         test/unit/tcframe/grader/GraderTests.cpp
         test/unit/tcframe/grader/MockGrader.hpp
         test/unit/tcframe/grader/MockGraderLogger.hpp
+        test/unit/tcframe/grader/MockGraderLoggerFactory.hpp
         test/unit/tcframe/grader/MockTestCaseGrader.hpp
         test/unit/tcframe/grader/TestCaseGraderTests.cpp
         test/unit/tcframe/io_manipulator/GridIOSegmentManipulatorTests.cpp

diff --git a/docs/api-ref/api-ref.rst b/docs/api-ref/api-ref.rst
@@ -633,3 +633,7 @@ Local grading
 .. py:function:: --no-memory-limit
 
     Unsets the memory limit specified by ``MemoryLimit()`` in grading config.
+
+.. py:function:: --brief
+
+    Makes the output of the local grading concise by only showing the verdicts.
diff --git a/docs/topic-guides/grading.rst b/docs/topic-guides/grading.rst
@@ -161,6 +161,44 @@ and here is for problems with subtasks.
 
 This local grading feature is useful for creating "unit tests" for your test cases. For each problem, you can write many solutions with different intended results. For example, ``solution_123.cpp`` should pass subtasks 1 - 3; ``solution_12.cpp`` should pass subtasks 1 and 2 but not subtask 3, etc.
 
+Brief mode
+----------
+
+You can pass an additional ``--brief`` argument to make the output concise. This is primarily intended to be consumed by scripts instead of human eyes.
+
+The first line of the output contains the overall the verdict in the following format:
+
+::
+
+    <status code> <points>
+
+where the status code mapping is:
+
+- ``AC``: Accepted
+- ``OK``: OK
+- ``WA``: Wrong Answer
+- ``RTE``: Runtime Error
+- ``TLE``: Time Limit Exceeded
+- ``ERR``: Internal Error
+
+
+If the problem has subtasks, the subtask verdicts will be output in the following lines, one line per subtask verdict ordered by subtask number, in the same format as above.
+
+The sample outputs from the previous sections would become the following using ``--brief`` argument:
+
+::
+
+    WA 71
+
+and
+
+::
+
+    RTE 40
+    AC 40
+    WA 0
+    RTE 0
+
 Notes
 -----
 

diff --git a/include/tcframe/grader.hpp b/include/tcframe/grader.hpp
@@ -1,7 +1,9 @@
 #pragma once
 
+#include "tcframe/grader/BriefGraderLogger.hpp"
 #include "tcframe/grader/DefaultGraderLogger.hpp"
 #include "tcframe/grader/Grader.hpp"
 #include "tcframe/grader/GraderLogger.hpp"
+#include "tcframe/grader/GraderLoggerFactory.hpp"
 #include "tcframe/grader/GradingOptions.hpp"
 #include "tcframe/grader/TestCaseGrader.hpp"
diff --git a/include/tcframe/grader/BriefGraderLogger.hpp b/include/tcframe/grader/BriefGraderLogger.hpp
@@ -0,0 +1,40 @@
+#pragma once
+
+#include <map>
+
+#include "GraderLogger.hpp"
+#include "tcframe/logger.hpp"
+#include "tcframe/verdict.hpp"
+
+using std::map;
+
+namespace tcframe {
+
+class BriefGraderLogger : public GraderLogger {
+private:
+    LoggerEngine* engine_;
+
+public:
+    virtual ~BriefGraderLogger() {}
+
+    BriefGraderLogger(LoggerEngine* engine)
+            : engine_(engine) {}
+
+    void logTestGroupIntroduction(int) {}
+    void logTestCaseIntroduction(const string&) {}
+    void logExecutionResults(const map<string, ExecutionResult>&) {}
+    void logIntroduction(const string&) {}
+    void logTestCaseVerdict(const Verdict&) {}
+
+    void logResult(const map<int, Verdict>& subtaskVerdicts, const Verdict& verdict) {
+        engine_->logParagraph(0, verdict.toBriefString());
+
+        if (subtaskVerdicts.size() > 1) {
+            for (auto entry : subtaskVerdicts) {
+                engine_->logParagraph(0, entry.second.toBriefString());
+            }
+        }
+    }
+};
+
+}
diff --git a/include/tcframe/grader/GraderLoggerFactory.hpp b/include/tcframe/grader/GraderLoggerFactory.hpp
@@ -0,0 +1,22 @@
+#pragma once
+
+#include "BriefGraderLogger.hpp"
+#include "DefaultGraderLogger.hpp"
+#include "GraderLogger.hpp"
+#include "tcframe/logger.hpp"
+
+namespace tcframe {
+
+class GraderLoggerFactory {
+public:
+    virtual ~GraderLoggerFactory() {}
+
+    virtual GraderLogger* create(LoggerEngine* engine, bool brief) {
+        if (brief) {
+            return new BriefGraderLogger(engine);
+        }
+        return new DefaultGraderLogger(engine);
+    }
+};
+
+}
diff --git a/include/tcframe/runner/Args.hpp b/include/tcframe/runner/Args.hpp
@@ -20,6 +20,7 @@ struct Args {
 private:
     Command command_;
 
+    bool brief_;
     optional<string> communicator_;
     optional<int> memoryLimit_;
     bool noMemoryLimit_;
@@ -32,13 +33,18 @@ struct Args {
 
 public:
     Args()
-            : noMemoryLimit_(false)
+            : brief_(false)
+            , noMemoryLimit_(false)
             , noTimeLimit_(false) {}
 
     Command command() const {
         return command_;
     }
 
+    bool brief() const {
+        return brief_;
+    }
+
     const optional<string>& communicator() const {
         return communicator_;
     }

diff --git a/include/tcframe/runner/ArgsParser.hpp b/include/tcframe/runner/ArgsParser.hpp
@@ -20,15 +20,16 @@ class ArgsParser {
 
     static Args parse(int argc, char* argv[]) {
         option longopts[] = {
-                { "communicator",    required_argument, nullptr, 'a'},
-                { "memory-limit",    required_argument, nullptr, 'b'},
-                { "no-memory-limit", no_argument      , nullptr, 'c'},
-                { "no-time-limit",   no_argument      , nullptr, 'd'},
-                { "output",          required_argument, nullptr, 'e'},
-                { "scorer",          required_argument, nullptr, 'f'},
-                { "seed",            required_argument, nullptr, 'g'},
-                { "solution",        required_argument, nullptr, 'h'},
-                { "time-limit",      required_argument, nullptr, 'i'},
+                { "brief",           no_argument,       nullptr, 'a'},
+                { "communicator",    required_argument, nullptr, 'b'},
+                { "memory-limit",    required_argument, nullptr, 'c'},
+                { "no-memory-limit", no_argument      , nullptr, 'd'},
+                { "no-time-limit",   no_argument      , nullptr, 'e'},
+                { "output",          required_argument, nullptr, 'f'},
+                { "scorer",          required_argument, nullptr, 'g'},
+                { "seed",            required_argument, nullptr, 'h'},
+                { "solution",        required_argument, nullptr, 'i'},
+                { "time-limit",      required_argument, nullptr, 'j'},
                 { 0, 0, 0, 0 }};
 
         Args args;
@@ -46,30 +47,33 @@ class ArgsParser {
         while ((c = getopt_long_only(argc, argv, ":", longopts, nullptr)) != -1) {
             switch (c) {
                 case 'a':
-                    args.communicator_ = optional<string>(optarg);
+                    args.brief_ = true;
                     break;
                 case 'b':
-                    args.memoryLimit_ = StringUtils::toNumber<int>(optarg);
+                    args.communicator_ = optional<string>(optarg);
                     break;
                 case 'c':
-                    args.noMemoryLimit_ = true;
+                    args.memoryLimit_ = StringUtils::toNumber<int>(optarg);
                     break;
                 case 'd':
-                    args.noTimeLimit_ = true;
+                    args.noMemoryLimit_ = true;
                     break;
                 case 'e':
-                    args.output_ = optional<string>(optarg);
+                    args.noTimeLimit_ = true;
                     break;
                 case 'f':
-                    args.scorer_ = optional<string>(optarg);
+                    args.output_ = optional<string>(optarg);
                     break;
                 case 'g':
-                    args.seed_ = StringUtils::toNumber<unsigned>(optarg);
+                    args.scorer_ = optional<string>(optarg);
                     break;
                 case 'h':
-                    args.solution_ = optional<string>(optarg);
+                    args.seed_ = StringUtils::toNumber<unsigned>(optarg);
                     break;
                 case 'i':
+                    args.solution_ = optional<string>(optarg);
+                    break;
+                case 'j':
                     args.timeLimit_ = StringUtils::toNumber<int>(optarg);
                     break;
                 case ':':

diff --git a/include/tcframe/runner/Runner.hpp b/include/tcframe/runner/Runner.hpp
@@ -31,6 +31,7 @@ class Runner {
     OperatingSystem* os_;
 
     RunnerLoggerFactory* runnerLoggerFactory_;
+    GraderLoggerFactory* graderLoggerFactory_;
     GeneratorFactory* generatorFactory_;
     GraderFactory* graderFactory_;
     EvaluatorRegistry* evaluatorRegistry_;
@@ -43,6 +44,7 @@ class Runner {
             LoggerEngine* loggerEngine,
             OperatingSystem* os,
             RunnerLoggerFactory* runnerLoggerFactory,
+            GraderLoggerFactory* graderLoggerFactory,
             GeneratorFactory* generatorFactory,
             GraderFactory* graderFactory,
             EvaluatorRegistry* evaluatorRegistry,
@@ -52,6 +54,7 @@ class Runner {
             , loggerEngine_(loggerEngine)
             , os_(os)
             , runnerLoggerFactory_(runnerLoggerFactory)
+            , graderLoggerFactory_(graderLoggerFactory)
             , generatorFactory_(generatorFactory)
             , graderFactory_(graderFactory)
             , evaluatorRegistry_(evaluatorRegistry)
@@ -154,7 +157,7 @@ class Runner {
 
         GradingOptions options = optionsBuilder.build();
 
-        auto logger = new DefaultGraderLogger(loggerEngine_);
+        auto logger = graderLoggerFactory_->create(loggerEngine_, args.brief());
         auto helperCommands = getHelperCommands(args, spec.styleConfig());
         auto evaluator = evaluatorRegistry_->get(spec.styleConfig().evaluationStyle(), os_, helperCommands);
         auto testCaseGrader = new TestCaseGrader(evaluator, logger);

diff --git a/include/tcframe/verdict/Verdict.hpp b/include/tcframe/verdict/Verdict.hpp
@@ -39,8 +39,20 @@ struct Verdict {
         return tie(status_, points_) == tie(o.status_, o.points_);
     }
 
+    string toBriefString() const {
+        string points = pointsToString();
+        if (!points.empty()) {
+            points = " " + points;
+        }
+        return status_.code() + points;
+    }
+
     string toString() const {
-        return status_.name() + pointsToString();
+        string points = pointsToString();
+        if (!points.empty()) {
+            points = " [" + points + "]";
+        }
+        return status_.name() + points;
     }
 
 private:
@@ -52,10 +64,10 @@ struct Verdict {
         while (points.back() == '0') {
             points.pop_back();
         }
-        while (points.back() == '.') {
+        if (points.back() == '.') {
             points.pop_back();
         }
-        return " [" + points + "]";
+        return points;
     }
 };
 

diff --git a/include/tcframe/verdict/VerdictStatus.hpp b/include/tcframe/verdict/VerdictStatus.hpp
@@ -45,6 +45,10 @@ struct VerdictStatus {
         return VerdictStatus("ERR", "Internal Error", 99);
     }
 
+    const string& code() const {
+        return code_;
+    }
+
     const string& name() const {
         return name_;
     }

diff --git a/src/tcframe/runner.cpp b/src/tcframe/runner.cpp
@@ -17,6 +17,7 @@ int main(int argc, char* argv[]) {
             new SimpleLoggerEngine(),
             new OperatingSystem(),
             new RunnerLoggerFactory(),
+            new GraderLoggerFactory(),
             new GeneratorFactory(),
             new GraderFactory(),
             new EvaluatorRegistry(new EvaluatorHelperRegistry()),

diff --git a/test/ete/resources/scripts/grade.sh b/test/ete/resources/scripts/grade.sh
@@ -5,4 +5,4 @@ set -ex
 export TCFRAME_HOME=../../tcframe
 
 g++ -o solution_alt solution_alt.cpp
-./runner grade --solution=./solution_alt
+./runner grade --solution=./solution_alt $@
diff --git a/test/ete/tcframe/GradingEteTests.cpp b/test/ete/tcframe/GradingEteTests.cpp
@@ -3,6 +3,7 @@
 #include "BaseEteTests.cpp"
 
 using ::testing::AllOf;
+using ::testing::Eq;
 using ::testing::HasSubstr;
 using ::testing::Test;
 
@@ -20,6 +21,11 @@ TEST_F(GradingEteTests, Normal) {
     EXPECT_THAT(result, HasSubstr("Time Limit Exceeded [33.33]"));
 }
 
+TEST_F(GradingEteTests, Normal_Brief) {
+    string result = exec("cd test-ete/normal && ../scripts/grade.sh --brief");
+    EXPECT_THAT(result, Eq("TLE 33.33\n"));
+}
+
 TEST_F(GradingEteTests, Normal_CustomScorer) {
     string result = exec("cd test-ete/normal-custom-scorer && ../scripts/grade-with-custom-scorer.sh");
     EXPECT_THAT(result, AllOf(

diff --git a/test/unit/tcframe/grader/BriefGraderLoggerTests.cpp b/test/unit/tcframe/grader/BriefGraderLoggerTests.cpp
@@ -0,0 +1,38 @@
+#include "gmock/gmock.h"
+#include "../mock.hpp"
+
+#include "../logger/MockLoggerEngine.hpp"
+#include "tcframe/grader/BriefGraderLogger.hpp"
+
+using ::testing::InSequence;
+using ::testing::Test;
+
+namespace tcframe {
+
+class BriefGraderLoggerTests : public Test {
+protected:
+    MOCK(LoggerEngine) engine;
+
+    BriefGraderLogger logger = BriefGraderLogger(&engine);
+};
+
+TEST_F(BriefGraderLoggerTests, Result) {
+    Verdict verdict(VerdictStatus::ac());
+    EXPECT_CALL(engine, logParagraph(0, verdict.toBriefString()));
+    logger.logResult({{Subtask::MAIN_ID, verdict}}, verdict);
+}
+
+TEST_F(BriefGraderLoggerTests, Result_WithSubtasks) {
+    Verdict verdict(VerdictStatus::wa(), 70);
+    Verdict subtask1Verdict(VerdictStatus::ac(), 70);
+    Verdict subtask2Verdict(VerdictStatus::wa(), 0);
+    {
+        InSequence sequence;
+        EXPECT_CALL(engine, logParagraph(0, verdict.toBriefString()));
+        EXPECT_CALL(engine, logParagraph(0, subtask1Verdict.toBriefString()));
+        EXPECT_CALL(engine, logParagraph(0, subtask2Verdict.toBriefString()));
+    }
+    logger.logResult({{1, subtask1Verdict}, {2, subtask2Verdict}}, verdict);
+}
+
+}