Refactor: finally have integration tests for UnixOperatingSystem

CMakeLists.txt

@@ -157,6 +157,7 @@ set(TEST_UNIT
         test/unit/tcframe/evaluator/BatchEvaluatorTests.cpp
         test/unit/tcframe/evaluator/MockEvaluator.hpp
         test/unit/tcframe/evaluator/MockEvaluatorRegistry.hpp
+        test/unit/tcframe/evaluator/scorer/CustomScorerTests.cpp
         test/unit/tcframe/evaluator/scorer/MockScorer.hpp
         test/unit/tcframe/generator/GeneratorLoggerTests.cpp
         test/unit/tcframe/generator/GeneratorTests.cpp
@@ -231,6 +232,7 @@ target_link_libraries(test_unit
 set(TEST_INTEGRATION
         test/integration/tcframe/evaluator/scorer/CustomScorerIntegrationTests.cpp
         test/integration/tcframe/evaluator/scorer/DiffScorerIntegrationTests.cpp
+        test/integration/tcframe/os/UnixOperatingSystemIntegrationTests.cpp
 )
 
 add_executable(test_integration ${INCLUDE} ${TEST_INTEGRATION})

include/tcframe/evaluator/scorer/CustomScorer.hpp

@@ -35,7 +35,6 @@ class CustomScorer : public Scorer {
         ExecutionResult executionResult = os_->execute(ExecutionRequestBuilder()
                 .setCommand(scoringCommand)
                 .setOutputFilename(SCORING_OUT_FILENAME)
-                .setErrorFilename(SCORING_ERR_FILENAME)
                 .build());
 
         Verdict verdict;

include/tcframe/evaluator/scorer/Scorer.hpp

@@ -11,7 +11,6 @@ namespace tcframe {
 class Scorer {
 public:
     static constexpr const char* SCORING_OUT_FILENAME = "_scoring.out";
-    static constexpr const char* SCORING_ERR_FILENAME = "_scoring.err";
 
     virtual ~Scorer() {}
 

include/tcframe/os/ExecutionRequest.hpp

@@ -19,7 +19,6 @@ struct ExecutionRequest {
     string command_;
     optional<string> inputFilename_;
     optional<string> outputFilename_;
-    optional<string> errorFilename_;
     optional<int> timeLimit_;
     optional<int> memoryLimit_;
 
@@ -36,10 +35,6 @@ struct ExecutionRequest {
         return outputFilename_;
     }
 
-    const optional<string>& errorFilename() const {
-        return errorFilename_;
-    }
-
     const optional<int>& timeLimit() const {
         return timeLimit_;
     }
@@ -49,8 +44,8 @@ struct ExecutionRequest {
     }
 
     bool operator==(const ExecutionRequest& o) const {
-        return tie(command_, inputFilename_, outputFilename_, errorFilename_, timeLimit_, memoryLimit_)
-                == tie(o.command_, o.inputFilename_, o.outputFilename_, o.errorFilename_, o.timeLimit_, o.memoryLimit_);
+        return tie(command_, inputFilename_, outputFilename_, timeLimit_, memoryLimit_)
+                == tie(o.command_, o.inputFilename_, o.outputFilename_, o.timeLimit_, o.memoryLimit_);
     }
 };
 
@@ -74,11 +69,6 @@ class ExecutionRequestBuilder {
         return *this;
     }
 
-    ExecutionRequestBuilder& setErrorFilename(string errorFilename) {
-        subject_.errorFilename_ = optional<string>(errorFilename);
-        return *this;
-    }
-
     ExecutionRequestBuilder& setTimeLimit(int timeLimit) {
         subject_.timeLimit_ = optional<int>(timeLimit);
         return *this;

include/tcframe/os/UnixOperatingSystem.hpp

@@ -71,22 +71,18 @@ class UnixOperatingSystem : public OperatingSystem {
             sout << " < " << request.inputFilename().value();
         }
         if (request.outputFilename()) {
-            sout << " > " << request.outputFilename().value();
+            sout << " 1> " << request.outputFilename().value();
         } else {
-            sout << " > /dev/null";
-        }
-        if (request.errorFilename()) {
-            sout << " 2> " << request.errorFilename().value();
-        } else {
-            sout << " 2> " << ERROR_FILENAME;
+            sout << " 1> /dev/null";
         }
+        sout << " 2> " << ERROR_FILENAME;
 
         ExecutionResultBuilder result;
 
         int exitValue = system(sout.str().c_str());
         int exitStatus = WEXITSTATUS(exitValue);
 
-        if (WIFSIGNALED(exitStatus)) {
+        if (exitStatus & (1<<7)) {
             int signal = WTERMSIG(exitStatus);
             result.setExitSignal(strsignal(signal));
             if (signal == SIGXCPU) {
@@ -96,12 +92,10 @@ class UnixOperatingSystem : public OperatingSystem {
             result.setExitCode(exitStatus);
         }
 
-        if (!request.errorFilename()) {
-            istream* errorStream = openForReading(ERROR_FILENAME);
-            string errorString = StringUtils::streamToString(errorStream);
-            closeOpenedStream(errorStream);
-            result.setStandardError(errorString);
-        }
+        istream* errorStream = openForReading(ERROR_FILENAME);
+        string errorString = StringUtils::streamToString(errorStream);
+        closeOpenedStream(errorStream);
+        result.setStandardError(errorString);
 
         return result.build();
     }

test/integration/resources/evaluator/scorer/custom/scorer.cpp

@@ -3,7 +3,6 @@ using namespace std;
 
 /*
  * AC if contestant's second line is a permutation of judge's.
- * OK 50 if not, but the largest elements are the same.
  * */
 
 int N;
@@ -12,16 +11,6 @@ vector<int> read(ifstream& out) {
     string answer;
     out >> answer;
 
-    if (answer == "crash") {
-        // division by zero
-        return vector<int>(1 / (N - 3));
-    }
-    if (answer == "bogus") {
-        // bogus output
-        cout << "bogus" << endl;
-        exit(0);
-    }
-
     vector<int> numbers(N);
     for (int i = 0; i < N; i++) {
         out >> numbers[i];
@@ -42,9 +31,6 @@ int main(int argc, char* argv[]) {
 
     if (ans_judge == ans_con) {
         cout << "AC" << endl;
-    } else if (ans_judge.back() == ans_con.back()) {
-        cout << "OK" << endl;
-        cout << "50" << endl;
     } else {
         cout << "WA" << endl;
     }

test/integration/resources/os/1.in

@@ -0,0 +1 @@
+42

test/integration/resources/os/program.cpp

@@ -0,0 +1,9 @@
+#include <bits/stdc++.h>
+using namespace std;
+
+int N;
+int main() {
+    cin >> N;
+    cout << N + 1 << endl;
+    cerr << N + 2 << endl;
+}

test/integration/resources/os/program_exit-code.cpp

@@ -0,0 +1,6 @@
+#include <bits/stdc++.h>
+using namespace std;
+
+int main() {
+    return 42;
+}

test/integration/resources/os/program_exit-signal.cpp

@@ -0,0 +1,9 @@
+#include <bits/stdc++.h>
+using namespace std;
+
+int main() {
+    vector<int> v;
+    int a = 1;
+
+    return a / (int)v.size();
+}

test/integration/resources/os/program_ml.cpp

@@ -0,0 +1,8 @@
+#include <bits/stdc++.h>
+using namespace std;
+
+int data[10000000];
+
+int main() {
+    return data[42];
+}

test/integration/resources/os/program_tl.cpp

@@ -0,0 +1,6 @@
+#include <bits/stdc++.h>
+using namespace std;
+
+int main() {
+    while (true);
+}

test/integration/tcframe/evaluator/scorer/CustomScorerIntegrationTests.cpp

@@ -4,6 +4,7 @@
 
 #include "tcframe/evaluator/scorer/CustomScorer.hpp"
 
+using ::testing::AllOf;
 using ::testing::Eq;
 using ::testing::HasSubstr;
 using ::testing::StartsWith;
@@ -26,44 +27,12 @@ class CustomScorerIntegrationTests : public Test {
     }
 };
 
-TEST_F(CustomScorerIntegrationTests, Scoring_AC) {
+TEST_F(CustomScorerIntegrationTests, Scoring) {
     ScoringResult result = scorer.score(
             "test-integration/evaluator/scorer/judge.in",
             "test-integration/evaluator/scorer/judge.out",
-            "test-integration/evaluator/scorer/custom/contestant_ac.out");
+            "test-integration/evaluator/scorer/custom/contestant.out");
     EXPECT_THAT(result.verdict(), Eq(Verdict(VerdictStatus::ac())));
 }
 
-TEST_F(CustomScorerIntegrationTests, Scoring_WA) {
-    ScoringResult result = scorer.score(
-            "test-integration/evaluator/scorer/judge.in",
-            "test-integration/evaluator/scorer/judge.out",
-            "test-integration/evaluator/scorer/custom/contestant_wa.out");
-    EXPECT_THAT(result.verdict(), Eq(Verdict(VerdictStatus::wa())));
-}
-
-TEST_F(CustomScorerIntegrationTests, Scoring_ERR_Crash) {
-    ScoringResult result = scorer.score(
-            "test-integration/evaluator/scorer/judge.in",
-            "test-integration/evaluator/scorer/judge.out",
-            "test-integration/evaluator/scorer/custom/contestant_err_crash.out");
-    EXPECT_THAT(result.verdict(), Eq(Verdict(VerdictStatus::err())));
-
-    ExecutionResult executionResult = result.executionResult();
-    EXPECT_FALSE(executionResult.isSuccessful());
-    EXPECT_TRUE(executionResult.exitSignal());
-}
-
-TEST_F(CustomScorerIntegrationTests, Scoring_ERR_Bogus) {
-    ScoringResult result = scorer.score(
-            "test-integration/evaluator/scorer/judge.in",
-            "test-integration/evaluator/scorer/judge.out",
-            "test-integration/evaluator/scorer/custom/contestant_err_bogus.out");
-    EXPECT_THAT(result.verdict(), Eq(Verdict(VerdictStatus::err())));
-
-    ExecutionResult executionResult = result.executionResult();
-    EXPECT_TRUE(executionResult.isSuccessful());
-    EXPECT_THAT(executionResult.standardError(), HasSubstr("Unknown verdict: bogus"));
-}
-
 }

test/integration/tcframe/os/UnixOperatingSystemIntegrationTests.cpp

@@ -0,0 +1,112 @@
+#include "gmock/gmock.h"
+
+#include <cstdlib>
+
+#include "tcframe/os/UnixOperatingSystem.hpp"
+#include "tcframe/util.hpp"
+
+using ::testing::Eq;
+using ::testing::Test;
+
+namespace tcframe {
+
+class UnixOperatingSystemIntegrationTests : public Test {
+protected:
+    UnixOperatingSystem os;
+
+    static string readFile(const string& path) {
+        ifstream in(path);
+        string contents = StringUtils::streamToString(&in);
+        in.close();
+        return contents;
+    }
+};
+
+TEST_F(UnixOperatingSystemIntegrationTests, Execution_Successful) {
+    system(
+            "g++ -o "
+            "test-integration/os/program "
+            "test-integration/os/program.cpp");
+
+    ExecutionResult result = os.execute(ExecutionRequestBuilder()
+            .setCommand("test-integration/os/program")
+            .setInputFilename("test-integration/os/1.in")
+            .setOutputFilename("test-integration/os/1.out")
+            .build());
+
+    EXPECT_TRUE(result.isSuccessful());
+    EXPECT_THAT(result.exitCode(), Eq(optional<int>(0)));
+    EXPECT_FALSE(result.exitSignal());
+
+    EXPECT_THAT(readFile("test-integration/os/1.out"), Eq("43\n"));
+    EXPECT_THAT(result.standardError(), Eq("44\n"));
+}
+
+TEST_F(UnixOperatingSystemIntegrationTests, Execution_ExceededTimeLimits) {
+    system(
+            "g++ -o "
+            "test-integration/os/program_tl "
+            "test-integration/os/program_tl.cpp");
+
+    ExecutionResult result = os.execute(ExecutionRequestBuilder()
+            .setCommand("test-integration/os/program_tl")
+            .setTimeLimit(1)
+            .build());
+
+    EXPECT_FALSE(result.isSuccessful());
+    EXPECT_FALSE(result.exitCode());
+    EXPECT_TRUE(result.exitSignal());
+    EXPECT_TRUE(result.exceededCpuLimits());
+}
+
+#ifndef __APPLE__
+
+TEST_F(UnixOperatingSystemIntegrationTests, Execution_ExceededMemoryLimits) {
+    system(
+            "g++ -o "
+            "test-integration/os/program_ml "
+            "test-integration/os/program_ml.cpp");
+
+    ExecutionResult result = os.execute(ExecutionRequestBuilder()
+            .setCommand("test-integration/os/program_ml")
+            .setMemoryLimit(16)
+            .build());
+
+    EXPECT_FALSE(result.isSuccessful());
+    EXPECT_FALSE(result.exitCode());
+    EXPECT_TRUE(result.exitSignal());
+}
+
+#endif
+
+TEST_F(UnixOperatingSystemIntegrationTests,  Execution_Crashed_ExitCode) {
+    system(
+            "g++ -o "
+            "test-integration/os/program_exit-code "
+            "test-integration/os/program_exit-code.cpp");
+
+    ExecutionResult result = os.execute(ExecutionRequestBuilder()
+            .setCommand("test-integration/os/program_exit-code")
+            .build());
+
+    EXPECT_FALSE(result.isSuccessful());
+    EXPECT_THAT(result.exitCode(), Eq(optional<int>(42)));
+    EXPECT_FALSE(result.exitSignal());
+}
+
+TEST_F(UnixOperatingSystemIntegrationTests, Execution_Crashed_ExitSignal) {
+    system(
+            "g++ -o "
+            "test-integration/os/program_exit-signal "
+            "test-integration/os/program_exit-signal.cpp");
+
+    ExecutionResult result = os.execute(ExecutionRequestBuilder()
+            .setCommand("test-integration/os/program_exit-signal")
+            .build());
+
+    EXPECT_FALSE(result.isSuccessful());
+    EXPECT_FALSE(result.exitCode());
+    EXPECT_TRUE(result.exitSignal());
+}
+
+}