Check and show errors on solution execution against input test case f…

…iles

Resolve #14

include/tcframe/exception.hpp

@@ -81,6 +81,12 @@ class SubtaskSatisfiabilityException : public TestCaseException {
             : TestCaseException(failures) { }
 };
 
+class ExecutionException : public TestCaseException {
+public:
+    ExecutionException(vector<Failure> failures)
+            : TestCaseException(failures) { }
+};
+
 }
 
 #endif

include/tcframe/generator.hpp

@@ -18,6 +18,7 @@
 #include <vector>
 
 using std::initializer_list;
+using std::istreambuf_iterator;
 using std::istringstream;
 using std::ostream;
 using std::set;
@@ -234,7 +235,15 @@ class BaseGenerator : protected TProblem, protected TestCasesCollector {
     }
 
     void generateTestCaseOutput(string testCaseInputName, string testCaseOutputName) {
-        os->execute(solutionExecutionCommand, testCaseInputName, testCaseOutputName);
+        ExecutionResult result = os->execute(solutionExecutionCommand, testCaseInputName, testCaseOutputName);
+
+        if (result.exitCode != 0) {
+            throw ExecutionException({
+                    Failure("Execution of solution failed:", 0),
+                    Failure("Exit code: " + Util::toString(result.exitCode), 1),
+                    Failure("Standard error: " + string(istreambuf_iterator<char>(*result.errorStream), istreambuf_iterator<char>()), 1)
+            });
+        }
     }
 };
 

include/tcframe/os.hpp

@@ -65,7 +65,8 @@ class UnixOperatingSystem : public OperatingSystem {
         string errorFilename = baseDirectoryName + "/_error.out";
 
         ExecutionResult result;
-        result.exitCode = system((command + " < " + inputFilename + " > " + outputFilename + " 2> " + errorFilename).c_str());
+        int exitStatus = system((command + " < " + inputFilename + " > " + outputFilename + " 2> " + errorFilename).c_str());
+        result.exitCode = WEXITSTATUS(exitStatus);
         result.outputStream = openForReading(outputFilename);
         result.errorStream = openForReadingAsStringStream(errorFilename);
 

test/exception_test.cpp

@@ -5,6 +5,7 @@
 #include <vector>
 
 using std::vector;
+using tcframe::ExecutionException;
 using tcframe::Failure;
 using tcframe::IOFormatException;
 using tcframe::ParsingException;
@@ -30,3 +31,10 @@ TEST(IOFormatException, Construction) {
     EXPECT_EQ((vector<Failure>{Failure("foo", 0)}), e.getFailures());
     EXPECT_EQ("", e.getMessage());
 }
+
+TEST(ExecutionException, Construction) {
+    ExecutionException e({Failure("foo", 0)});
+
+    EXPECT_EQ((vector<Failure>{Failure("foo", 0)}), e.getFailures());
+    EXPECT_EQ("", e.getMessage());
+}

test/generator_test.cpp

@@ -3,13 +3,15 @@
 #include "tcframe/generator.hpp"
 
 #include <map>
+#include <set>
 #include <sstream>
 #include <string>
 #include <vector>
 
 using std::istringstream;
 using std::map;
 using std::ostringstream;
+using std::set;
 using std::string;
 using std::vector;
 using tcframe::BaseGenerator;
@@ -51,6 +53,11 @@ class FakeLogger : public Logger {
 
 class FakeOperatingSystem : public OperatingSystem {
 public:
+    FakeOperatingSystem() { }
+
+    FakeOperatingSystem(set<string> arrangedFailedInputNames)
+            : arrangedFailedInputNames(arrangedFailedInputNames) { }
+
     void setBaseDirectory(string) override { }
 
     istream* openForReading(string) override {
@@ -66,6 +73,10 @@ class FakeOperatingSystem : public OperatingSystem {
     void remove(string) override { }
 
     ExecutionResult execute(string, string inputName, string) override {
+        if (arrangedFailedInputNames.count(inputName)) {
+            return ExecutionResult{1, new istringstream(), new istringstream("Intentionally failed")};
+        }
+
         istringstream input(testCaseInputs[inputName]->str());
 
         int A, B, K;
@@ -75,14 +86,15 @@ class FakeOperatingSystem : public OperatingSystem {
 
         ExecutionResult result;
         result.exitCode = 0;
-        result.errorStream = new istringstream();
         result.outputStream = new istringstream(output);
+        result.errorStream = new istringstream();
 
         return result;
     }
 
 private:
     map<string, ostringstream*> testCaseInputs;
+    set<string> arrangedFailedInputNames;
 };
 
 class ProblemWithSubtasks : public BaseProblem {
@@ -204,14 +216,26 @@ class GeneratorForInvalidInputFormatProblem : public BaseGenerator<ProblemWithIn
 protected:
     void Config() { }
 
-    void TestCases() {
+    void TestCases() { }
 
-    }
 public:
     GeneratorForInvalidInputFormatProblem(FakeLogger* logger)
             : BaseGenerator(logger, new FakeOperatingSystem()) { }
 };
 
+class GeneratorWithArrangedExecutionFailure : public BaseGenerator<ProblemWithoutSubtasks> {
+protected:
+    void Config() { }
+
+    void TestCases() {
+        addOfficialTestCase([this] { A = 1, B = 100, K = 7; }, "A = 1, B = 100, K = 7");
+    }
+
+public:
+    GeneratorWithArrangedExecutionFailure(FakeLogger* logger, set<string> arrangedFailedInputNames)
+            : BaseGenerator(logger, new FakeOperatingSystem(arrangedFailedInputNames)) { }
+};
+
 TEST(GeneratorTest, GenerationWithSubtasksAndTestGroups) {
     FakeLogger logger;
     GeneratorWithTestGroups gen(&logger);
@@ -279,3 +303,17 @@ TEST(GeneratorTest, GenerationWithInvalidInputFormat) {
     ASSERT_EQ(1, failures.size());
     EXPECT_EQ(Failure("Variable type of `K` unsatisfied. Expected: basic scalar or string type, or vector of those types", 0), failures[0]);
 }
+
+TEST(GeneratorTest, GenerationWithFailedExecution) {
+    FakeLogger logger;
+    GeneratorWithArrangedExecutionFailure gen(&logger, {"problem_1.in"});
+    int exitCode = gen.generate();
+
+    EXPECT_NE(0, exitCode);
+
+    auto failures = logger.getFailures("problem_1");
+    ASSERT_EQ(3, failures.size());
+    EXPECT_EQ(Failure("Execution of solution failed:", 0), failures[0]);
+    EXPECT_EQ(Failure("Exit code: 1", 1), failures[1]);
+    EXPECT_EQ(Failure("Standard error: Intentionally failed", 1), failures[2]);
+}