Skip to content

Commit

Permalink
Fix regression when the final verdict of problem without subtasks is …
Browse files Browse the repository at this point in the history 
…always AC (#107)

This was introduced by the global constraints feature in 1.0.
The problem is we were using -1 as the key of subtask ID in the verdict map,
but we removed it.
  • Loading branch information
@fushar
fushar committed Dec 14, 2016
1 parent b3fa95a commit fc23480
Show file tree
Hide file tree
Showing 6 changed files with 76 additions and 25 deletions.
19 changes: 17 additions & 2 deletions include/tcframe/grader/Grader.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "GraderLogger.hpp"
#include "TestCaseGrader.hpp"
#include "tcframe/os.hpp"
#include "tcframe/spec.hpp"
#include "tcframe/util.hpp"
#include "tcframe/verdict.hpp"

Expand All @@ -33,11 +34,11 @@ class Grader {
: testCaseGrader_(testCaseGrader)
, logger_(logger) {}

virtual void grade(const TestSuite& testSuite, const set<int>& subtaskIds, const GraderConfig& config) {
virtual void grade(const TestSuite& testSuite, const ConstraintSuite& constraintSuite, const GraderConfig& config) {
logger_->logIntroduction();

map<int, Verdict> subtaskVerdicts;
for (int subtaskId : subtaskIds) {
for (int subtaskId : getSubtaskIdsToGrade(constraintSuite)) {
subtaskVerdicts[subtaskId] = Verdict::ac();
}

Expand All @@ -49,6 +50,20 @@ class Grader {
}

private:
set<int> getSubtaskIdsToGrade(const ConstraintSuite& constraintSuite) {
set<int> subtaskIds;
for (const Subtask& subtask : constraintSuite.constraints()) {
subtaskIds.insert(subtask.id());
}

// remove global constraints for problem with subtasks
if (subtaskIds.size() > 1) {
subtaskIds.erase(-1);
}

return subtaskIds;
}

void gradeOnTestGroup(
const TestGroup& testGroup,
const GraderConfig& config,
Expand Down
7 changes: 1 addition & 6 deletions include/tcframe/runner/Runner.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,12 +149,7 @@ class Runner {
auto testCaseGrader = new TestCaseGrader(evaluator, scorer, logger);
auto grader = graderFactory_->create(testCaseGrader, logger);

set<int> subtaskIds;
for (const Subtask& subtask : spec.constraintSuite().constraints()) {
subtaskIds.insert(subtask.id());
}

grader->grade(spec.testSuite(), subtaskIds, graderConfig);
grader->grade(spec.testSuite(), spec.constraintSuite(), graderConfig);
return 0;
}

Expand Down
6 changes: 3 additions & 3 deletions include/tcframe/spec/testcase/TestSuite.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ class TestSuiteBuilder {
TestSuiteBuilder()
: beforeClosure_([]{})
, afterClosure_([]{})
, curSampleSubtaskIds_({})
, curOfficialSubtaskIds_({})
, curSampleSubtaskIds_({-1})
, curOfficialSubtaskIds_({-1})
, curOfficialTestGroupId_(-1)
, hasCurOfficialTestGroup_(false)
, hasCurSampleTestCase_(false) {
Expand All @@ -95,7 +95,7 @@ class TestSuiteBuilder {
}

hasCurSampleTestCase_ = true;
curSampleSubtaskIds_ = {};
curSampleSubtaskIds_ = {-1};
curSubtaskIds_ = &curSampleSubtaskIds_;
curSampleInputLines_ = optional<vector<string>>();
curSampleOutputLines_ = optional<vector<string>>();
Expand Down
51 changes: 46 additions & 5 deletions test/tcframe/grader/GraderTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ class GraderTests : public Test {
MOCK(TestCaseGrader) testCaseGrader;
MOCK(GraderLogger) logger;

TestCase stcA = TestUtils::createFakeTestCase("foo_sample_1");
TestCase tcA = TestUtils::createFakeTestCase("foo_1");
TestCase tcB = TestUtils::createFakeTestCase("foo_2");

TestCase stc1 = TestUtils::createFakeTestCase("foo_sample_1", {1, 2});
TestCase stc2 = TestUtils::createFakeTestCase("foo_sample_2", {2});
TestCase tc1 = TestUtils::createFakeTestCase("foo_1_1", {1, 2});
Expand All @@ -31,14 +35,25 @@ class GraderTests : public Test {
TestCase tc4 = TestUtils::createFakeTestCase("foo_4_1", {4});
TestCase tc5 = TestUtils::createFakeTestCase("foo_4_2", {4});

set<int> subtaskIds = {1, 2, 3, 4};

TestSuite testSuite = TestSuite({
TestGroup(0, {stcA}),
TestGroup(-1, {tcA, tcB})});
ConstraintSuite constraintSuite = ConstraintSuite(
{Subtask(-1, {})},
{});

TestSuite testSuiteWithSubtasks = TestSuite({
TestGroup(0, {stc1, stc2}),
TestGroup(1, {tc1}),
TestGroup(2, {tc2}),
TestGroup(3, {tc3}),
TestGroup(4, {tc4, tc5})});
ConstraintSuite constraintSuiteWithSubtasks = ConstraintSuite(
{Subtask(1, {}), Subtask(2, {}), Subtask(3, {}), Subtask(4, {})},
{});
ConstraintSuite constraintSuiteWithSubtasksWithGlobalConstraints = ConstraintSuite(
{Subtask(-1, {}), Subtask(1, {}), Subtask(2, {}), Subtask(3, {}), Subtask(4, {})},
{});

GraderConfig config = GraderConfigBuilder("foo")
.setSolutionCommand("python Sol.py")
Expand Down Expand Up @@ -66,6 +81,22 @@ class GraderTests : public Test {
int GraderTests::T;

TEST_F(GraderTests, Grading) {
{
InSequence sequence;
EXPECT_CALL(logger, logIntroduction());
EXPECT_CALL(logger, logTestGroupIntroduction(0));
EXPECT_CALL(testCaseGrader, grade(stcA, config));
EXPECT_CALL(logger, logTestGroupIntroduction(-1));
EXPECT_CALL(testCaseGrader, grade(tcA, config));
EXPECT_CALL(testCaseGrader, grade(tcB, config));

EXPECT_CALL(logger, logResult(map<int, Verdict>{
{-1, Verdict::ac()}}));
}
grader.grade(testSuite, constraintSuite, config);
}

TEST_F(GraderTests, Grading_WithSubtasks) {
{
InSequence sequence;
EXPECT_CALL(logger, logIntroduction());
Expand All @@ -88,10 +119,20 @@ TEST_F(GraderTests, Grading) {
{3, Verdict::wa()},
{4, Verdict::tle()}}));
}
grader.grade(testSuite, subtaskIds, config);
grader.grade(testSuiteWithSubtasks, constraintSuiteWithSubtasks, config);
}

TEST_F(GraderTests, Grading_WithSubtasks_WithGlobalConstraints) {
EXPECT_CALL(logger, logResult(map<int, Verdict>{
{1, Verdict::ac()},
{2, Verdict::rte()},
{3, Verdict::wa()},
{4, Verdict::tle()}}));

grader.grade(testSuiteWithSubtasks, constraintSuiteWithSubtasksWithGlobalConstraints, config);
}

TEST_F(GraderTests, Grading_MultipleTestCases) {
TEST_F(GraderTests, Grading_WithSubtasks_MultipleTestCases) {
{
InSequence sequence;
EXPECT_CALL(logger, logIntroduction());
Expand All @@ -117,7 +158,7 @@ TEST_F(GraderTests, Grading_MultipleTestCases) {
{3, Verdict::wa()},
{4, Verdict::tle()}}));
}
grader.grade(testSuite, subtaskIds, multipleTestCasesConfig);
grader.grade(testSuiteWithSubtasks, constraintSuiteWithSubtasks, multipleTestCasesConfig);
}

}
2 changes: 1 addition & 1 deletion test/tcframe/grader/MockGrader.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ class MockGrader : public Grader {
MockGrader()
: Grader(nullptr, nullptr) {}

MOCK_METHOD3(grade, void(const TestSuite&, const set<int>&, const GraderConfig&));
MOCK_METHOD3(grade, void(const TestSuite&, const ConstraintSuite&, const GraderConfig&));
};

class MockGraderFactory : public GraderFactory {
Expand Down
16 changes: 8 additions & 8 deletions test/tcframe/spec/testcase/TestSuiteBuilderTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,12 @@ TEST_F(TestSuiteBuilderTests, Building_OnlySample) {
TestGroup(0, {
TestCaseBuilder()
.setId("foo_sample_1")
.setSubtaskIds({})
.setSubtaskIds({-1})
.setData(new SampleTestCaseData("10\n20\n", "yes\n"))
.build(),
TestCaseBuilder()
.setId("foo_sample_2")
.setSubtaskIds({})
.setSubtaskIds({-1})
.setData(new SampleTestCaseData("30\n"))
.build()})});

Expand All @@ -82,13 +82,13 @@ TEST_F(TestSuiteBuilderTests, Building_OnlyOfficial) {
TestGroup(-1, {
TestCaseBuilder()
.setId("foo_1")
.setSubtaskIds({})
.setSubtaskIds({-1})
.setDescription("N = 1")
.setData(new OfficialTestCaseData([]{}))
.build(),
TestCaseBuilder()
.setId("foo_2")
.setSubtaskIds({})
.setSubtaskIds({-1})
.setDescription("N = 2")
.setData(new OfficialTestCaseData([]{}))
.build()})});
Expand Down Expand Up @@ -120,24 +120,24 @@ TEST_F(TestSuiteBuilderTests, Building_Both) {
TestGroup(0, {
TestCaseBuilder()
.setId("foo_sample_1")
.setSubtaskIds({})
.setSubtaskIds({-1})
.setData(new SampleTestCaseData("10\n20\n", "yes\n"))
.build(),
TestCaseBuilder()
.setId("foo_sample_2")
.setSubtaskIds({})
.setSubtaskIds({-1})
.setData(new SampleTestCaseData("30\n"))
.build()}),
TestGroup(-1, {
TestCaseBuilder()
.setId("foo_1")
.setSubtaskIds({})
.setSubtaskIds({-1})
.setDescription("N = 1")
.setData(new OfficialTestCaseData([]{}))
.build(),
TestCaseBuilder()
.setId("foo_2")
.setSubtaskIds({})
.setSubtaskIds({-1})
.setDescription("N = 2")
.setData(new OfficialTestCaseData([]{}))
.build()})});
Expand Down

0 comments on commit fc23480

Please sign in to comment.