Skip to content

Commit

Permalink
Grading refactor: introduce TestCaseGrade (#127)
Browse files Browse the repository at this point in the history
This class holds the evaluation and scoring results, as well as
the final verdict.

The term "message" in scoring result is also renamed to "private info",
which can only be seen by judges.

This is a preparation for supporting scores in grade.
  • Loading branch information
fushar committed Mar 11, 2017
1 parent 7dfb58c commit e754212
Show file tree
Hide file tree
Showing 39 changed files with 587 additions and 340 deletions.
15 changes: 9 additions & 6 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ include_directories(include)
set(INCLUDE
include/tcframe/evaluator.hpp
include/tcframe/evaluator/BatchEvaluator.hpp
include/tcframe/evaluator/EvaluationResult.hpp
include/tcframe/evaluator/Evaluator.hpp
include/tcframe/evaluator/EvaluatorConfig.hpp
include/tcframe/generator.hpp
Expand All @@ -53,6 +52,12 @@ set(INCLUDE
include/tcframe/generator/GeneratorConfig.hpp
include/tcframe/generator/GeneratorLogger.hpp
include/tcframe/generator/TestCaseGenerator.hpp
include/tcframe/grade.hpp
include/tcframe/grade/EvaluationResult.hpp
include/tcframe/grade/ScoringResult.hpp
include/tcframe/grade/TestCaseGrade.hpp
include/tcframe/grade/TestCaseGradeCreator.hpp
include/tcframe/grade/Verdict.hpp
include/tcframe/grader.hpp
include/tcframe/grader/Grader.hpp
include/tcframe/grader/GraderConfig.hpp
Expand Down Expand Up @@ -87,7 +92,7 @@ set(INCLUDE
include/tcframe/scorer/CustomScorer.hpp
include/tcframe/scorer/DiffScorer.hpp
include/tcframe/scorer/Scorer.hpp
include/tcframe/scorer/ScoringResult.hpp
include/tcframe/scorer/ScoringResultParser.hpp
include/tcframe/spec.hpp
include/tcframe/spec/constraint.hpp
include/tcframe/spec/constraint/Constraint.hpp
Expand Down Expand Up @@ -132,9 +137,6 @@ set(INCLUDE
include/tcframe/util/NotImplementedException.hpp
include/tcframe/util/StringUtils.hpp
include/tcframe/util/optional.hpp
include/tcframe/verdict.hpp
include/tcframe/verdict/Verdict.hpp
include/tcframe/verdict/VerdictParser.hpp
include/tcframe/verifier.hpp
include/tcframe/verifier/ConstraintsVerificationResult.hpp
include/tcframe/verifier/MultipleTestCasesConstraintsVerificationResult.hpp
Expand Down Expand Up @@ -171,6 +173,7 @@ set(TEST_UNIT
test/unit/tcframe/runner/RunnerLoggerTests.cpp
test/unit/tcframe/runner/SlugParserTests.cpp
test/unit/tcframe/scorer/MockScorer.hpp
test/unit/tcframe/scorer/ScoringResultParserTests.cpp
test/unit/tcframe/spec/constraint/ConstraintSuiteBuilderTests.cpp
test/unit/tcframe/spec/core/BaseTestSpecTests.cpp
test/unit/tcframe/spec/core/BaseProblemSpecTests.cpp
Expand All @@ -191,6 +194,7 @@ set(TEST_UNIT
test/unit/tcframe/spec/variable/VariableTests.cpp
test/unit/tcframe/spec/variable/VectorTests.cpp
test/unit/tcframe/spec/variable/WhitespaceManipulatorTests.cpp
test/unit/tcframe/grade/TestCaseGradeCreatorTests.cpp
test/unit/tcframe/grader/GraderTests.cpp
test/unit/tcframe/grader/GraderLoggerTests.cpp
test/unit/tcframe/grader/MockGrader.hpp
Expand All @@ -200,7 +204,6 @@ set(TEST_UNIT
test/unit/tcframe/util/OptionalTests.cpp
test/unit/tcframe/util/StringUtilsTests.cpp
test/unit/tcframe/util/TestUtils.hpp
test/unit/tcframe/verdict/VerdictParserTests.cpp
test/unit/tcframe/verifier/MockVerifier.hpp
test/unit/tcframe/verifier/VerifierTests.cpp
)
Expand Down
1 change: 0 additions & 1 deletion include/tcframe/evaluator.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#pragma once

#include "tcframe/evaluator/BatchEvaluator.hpp"
#include "tcframe/evaluator/EvaluationResult.hpp"
#include "tcframe/evaluator/Evaluator.hpp"
3 changes: 1 addition & 2 deletions include/tcframe/evaluator/BatchEvaluator.hpp
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
#pragma once

#include "Evaluator.hpp"
#include "EvaluationResult.hpp"
#include "EvaluatorConfig.hpp"
#include "tcframe/grade.hpp"
#include "tcframe/os.hpp"
#include "tcframe/util.hpp"
#include "tcframe/verdict.hpp"

namespace tcframe {

Expand Down
4 changes: 3 additions & 1 deletion include/tcframe/evaluator/Evaluator.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,17 @@

#include <string>

#include "EvaluationResult.hpp"
#include "EvaluatorConfig.hpp"
#include "tcframe/grade.hpp"

using std::string;

namespace tcframe {

class Evaluator {
public:
static constexpr const char* EVALUATION_FILENAME = "_evaluation.out";

virtual ~Evaluator() {}

virtual EvaluationResult evaluate(
Expand Down
5 changes: 1 addition & 4 deletions include/tcframe/generator/GeneratorLogger.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,8 @@ class GeneratorLogger : public BaseLogger {
}
}

virtual void logSampleTestCaseCheckFailure(const string& scoringMessage) {
virtual void logSampleTestCaseCheckFailure() {
engine_->logListItem1(2, "Sample test case output does not match with actual output produced by the solution");
if (!scoringMessage.empty()) {
engine_->logListItem2(3, scoringMessage);
}
}

virtual void logSampleTestCaseNoOutputNeededFailure() {
Expand Down
25 changes: 13 additions & 12 deletions include/tcframe/generator/TestCaseGenerator.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@
#include "GeneratorConfig.hpp"
#include "GeneratorLogger.hpp"
#include "tcframe/evaluator.hpp"
#include "tcframe/grade.hpp"
#include "tcframe/io_manipulator.hpp"
#include "tcframe/os.hpp"
#include "tcframe/scorer.hpp"
#include "tcframe/spec.hpp"
#include "tcframe/verdict.hpp"
#include "tcframe/verifier.hpp"

using std::char_traits;
Expand Down Expand Up @@ -127,9 +127,11 @@ class TestCaseGenerator {
.build();

EvaluationResult evaluationResult = evaluator_->evaluate(inputFilename, outputFilename, evaluatorConfig);
ExecutionResult executionResult = evaluationResult.executionResult();
if (!executionResult.isSuccessful()) {
throw GenerationException([=] { logger_->logExecutionFailure("solution", executionResult); });
if (!evaluationResult.executionResult().isSuccessful()) {
TestCaseGrade grade = TestCaseGradeCreator()
.setEvaluationResult(evaluationResult)
.create();
throw GenerationException([=] { logger_->logTestCaseGradeDetails(grade); });
}

if (maybeSampleOutputString) {
Expand Down Expand Up @@ -185,19 +187,18 @@ class TestCaseGenerator {
string modifiedSampleOutputString = sampleOutputString;
modifySampleOutputStringForMultipleTestCases(modifiedSampleOutputString, config);

ostream* sampleOutput = os_->openForWriting("_evaluation.out");
ostream* sampleOutput = os_->openForWriting(Evaluator::EVALUATION_FILENAME);
*sampleOutput << modifiedSampleOutputString;
os_->closeOpenedStream(sampleOutput);

ScoringResult scoringResult = scorer_->score(inputFilename, outputFilename, "_evaluation.out");
ScoringResult scoringResult = scorer_->score(inputFilename, outputFilename, Evaluator::EVALUATION_FILENAME);
TestCaseGrade grade = TestCaseGradeCreator()
.setScoringResult(scoringResult)
.create();
if (!(scoringResult.verdict() == Verdict::ac())) {
throw GenerationException([=] {
logger_->logSampleTestCaseCheckFailure(scoringResult.message());
if (scoringResult.executionResult().isSuccessful()) {
logger_->logTestCaseScoringMessage(scoringResult.message());
} else {
logger_->logExecutionFailure("scorer", scoringResult.executionResult());
}
logger_->logSampleTestCaseCheckFailure();
logger_->logTestCaseGradeDetails(grade);
});
}
}
Expand Down
5 changes: 5 additions & 0 deletions include/tcframe/grade.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#include "tcframe/grade/EvaluationResult.hpp"
#include "tcframe/grade/ScoringResult.hpp"
#include "tcframe/grade/TestCaseGrade.hpp"
#include "tcframe/grade/TestCaseGradeCreator.hpp"
#include "tcframe/grade/Verdict.hpp"
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
#include <tuple>
#include <utility>

#include "Verdict.hpp"
#include "tcframe/os.hpp"
#include "tcframe/util.hpp"
#include "tcframe/verdict.hpp"

using std::move;
using std::string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
#include <tuple>
#include <utility>

#include "Verdict.hpp"
#include "tcframe/os.hpp"
#include "tcframe/verdict.hpp"
#include "tcframe/util.hpp"

using std::move;
using std::string;
Expand All @@ -17,25 +18,26 @@ struct ScoringResult {
friend class ScoringResultBuilder;

private:
ExecutionResult executionResult_;
optional<ExecutionResult> executionResult_;
Verdict verdict_;
string message_;
optional<string> privateInfo_;

public:
const ExecutionResult& executionResult() const {
const optional<ExecutionResult>& executionResult() const {
return executionResult_;
}

const Verdict& verdict() const {
return verdict_;
}

const string& message() const {
return message_;
const optional<string>& privateInfo() const {
return privateInfo_;
}

bool operator==(const ScoringResult& o) const {
return tie(executionResult_, verdict_, message_) == tie(o.executionResult_, o.verdict_, o.message_);
return tie(executionResult_, verdict_, privateInfo_)
== tie(o.executionResult_, o.verdict_, o.privateInfo_);
}
};

Expand All @@ -45,7 +47,7 @@ class ScoringResultBuilder {

public:
ScoringResultBuilder& setExecutionResult(ExecutionResult executionResult) {
subject_.executionResult_ = executionResult;
subject_.executionResult_ = optional<ExecutionResult>(executionResult);
return *this;
}

Expand All @@ -54,8 +56,8 @@ class ScoringResultBuilder {
return *this;
}

ScoringResultBuilder& setMessage(string message) {
subject_.message_ = message;
ScoringResultBuilder& setPrivateInfo(string privateInfo) {
subject_.privateInfo_ = privateInfo;
return *this;
}

Expand Down
111 changes: 111 additions & 0 deletions include/tcframe/grade/TestCaseGrade.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#pragma once

#include <string>
#include <tuple>
#include <utility>

#include "Verdict.hpp"
#include "tcframe/os.hpp"
#include "tcframe/util.hpp"

using std::move;
using std::string;
using std::tie;

namespace tcframe {

struct TestCaseGrade {
friend class TestCaseGradeBuilder;

private:
Verdict verdict_;
optional<double> score_;

optional<ExecutionResult> evaluationExecutionResult_;
optional<ExecutionResult> scoringExecutionResult_;
optional<string> privateScoringInfo_;

public:
const Verdict& verdict() const {
return verdict_;
}

const optional<double>& score() const {
return score_;
}

optional<ExecutionResult> evaluationExecutionResult() const {
return evaluationExecutionResult_;
}

optional<ExecutionResult> scoringExecutionResult() const {
return scoringExecutionResult_;
}

const optional<string>& privateScoringInfo() const {
return privateScoringInfo_;
}

bool operator==(const TestCaseGrade& o) const {
return tie(
verdict_,
score_,
evaluationExecutionResult_,
scoringExecutionResult_,
privateScoringInfo_) == tie(
o.verdict_,
o.score_,
o.evaluationExecutionResult_,
o.scoringExecutionResult_,
o.privateScoringInfo_);
}
};

class TestCaseGradeBuilder {
private:
TestCaseGrade subject_;

public:
TestCaseGradeBuilder& setVerdict(Verdict verdict) {
subject_.verdict_ = verdict;
return *this;
}

TestCaseGradeBuilder& setScore(double score) {
subject_.score_ = optional<double>(score);
return *this;
}

TestCaseGradeBuilder& setEvaluationExecutionResult(ExecutionResult evaluationExecutionResult) {
return setEvaluationExecutionResult(optional<ExecutionResult>(evaluationExecutionResult));
}

TestCaseGradeBuilder& setEvaluationExecutionResult(optional<ExecutionResult> evaluationExecutionResult) {
subject_.evaluationExecutionResult_ = evaluationExecutionResult;
return *this;
}

TestCaseGradeBuilder& setScoringExecutionResult(ExecutionResult scoringExecutionResult) {
return setScoringExecutionResult(optional<ExecutionResult>(scoringExecutionResult));
}

TestCaseGradeBuilder& setScoringExecutionResult(optional<ExecutionResult> scoringExecutionResult) {
subject_.scoringExecutionResult_ = scoringExecutionResult;
return *this;
}

TestCaseGradeBuilder& setPrivateScoringInfo(string privateScoringInfo) {
return setPrivateScoringInfo(optional<string>(privateScoringInfo));
}

TestCaseGradeBuilder& setPrivateScoringInfo(optional<string> privateScoringInfo) {
subject_.privateScoringInfo_ = privateScoringInfo;
return *this;
}

TestCaseGrade build() {
return move(subject_);
}
};

}

0 comments on commit e754212

Please sign in to comment.