Merge pull request #18 from MichailKon/dev/invoker/pipeline

Refactor of invoker compile and run pipeline

Author: grphil

common/config/invoker.go

@@ -22,7 +22,7 @@ type InvokerConfig struct {
 
 	CompilerConfigsFolder string `yaml:"CompilerConfigsFolder"`
 
-	CheckerLimits *RunConfig `yaml:"CheckerLimits,omitempty"`
+	CheckerLimits *RunLimitsConfig `yaml:"CheckerLimits,omitempty"`
 }
 
 func FillInInvokerConfig(config *InvokerConfig) {
@@ -63,7 +63,7 @@ func FillInInvokerConfig(config *InvokerConfig) {
 	}
 
 	if config.CheckerLimits == nil {
-		config.CheckerLimits = &RunConfig{}
+		config.CheckerLimits = &RunLimitsConfig{}
 	}
-	fillInDefaultCheckerRunConfig(config.CheckerLimits)
+	fillInDefaultCheckerRunLimitsConfig(config.CheckerLimits)
 }

common/config/run_config.go renamed to common/config/run_limits_config.go

@@ -2,7 +2,7 @@ package config
 
 import "testing_system/lib/customfields"
 
-type RunConfig struct {
+type RunLimitsConfig struct {
 	TimeLimit     customfields.Time   `yaml:"TimeLimit" json:"TimeLimit"`
 	MemoryLimit   customfields.Memory `yaml:"MemoryLimit" json:"MemoryLimit"`
 	WallTimeLimit customfields.Time   `yaml:"WallTimeLimit" json:"WallTimeLimit"`
@@ -21,7 +21,7 @@ type RunConfig struct {
 	MaxOutputSize customfields.Memory `yaml:"MaxOutputSize" json:"MaxOutputSize"`
 }
 
-func fillInDefaultCheckerRunConfig(config *RunConfig) {
+func fillInDefaultCheckerRunLimitsConfig(config *RunLimitsConfig) {
 	if config.TimeLimit == 0 {
 		config.TimeLimit.FromStr("15s")
 	}

common/constants/verdict/verdict.go

@@ -8,6 +8,7 @@ const (
 	PT Verdict = "PT" // Partial solution
 
 	WA Verdict = "WA" // Wrong answer
+	WR Verdict = "WR" // Wrong (used only for interactive problems)
 
 	RT Verdict = "RT" // Runtime error
 	ML Verdict = "ML" // Memory limit

invoker/check_pipeline.go

@@ -0,0 +1,197 @@
+package invoker
+
+import (
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"golang.org/x/net/html/charset"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing_system/common/connectors/storageconn"
+	"testing_system/common/constants/resource"
+	"testing_system/common/constants/verdict"
+	"testing_system/invoker/sandbox"
+	"testing_system/lib/logger"
+)
+
+func (s *JobPipelineState) fullCheckPipeline() error {
+	err := s.loadCheckerBinaryFile()
+	if err != nil {
+		return err
+	}
+
+	err = s.loadTestAnswerFile()
+	if err != nil {
+		return err
+	}
+
+	err = s.generateCheckerRunConfig()
+	if err != nil {
+		return err
+	}
+
+	err = s.executeCheckerRunCommand()
+	if err != nil {
+		return err
+	}
+
+	err = s.parseCheckerResult()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *JobPipelineState) generateCheckerRunConfig() error {
+	s.test.checkConfig = &sandbox.ExecuteConfig{
+		RunLimitsConfig: *s.invoker.TS.Config.Invoker.CheckerLimits,
+	}
+
+	s.test.checkConfig.Command = checkerBinaryFile
+	s.test.checkConfig.Args = []string{
+		testInputFile, testOutputFile, testAnswerFile, checkResultFile, checkResultFileArg,
+	}
+	logger.Trace("Generated checker run config for %s", s.loggerData)
+	return nil
+}
+
+func (s *JobPipelineState) executeCheckerRunCommand() error {
+	s.executeWaitGroup.Add(1)
+	s.invoker.RunQueue <- s.runChecker
+	s.executeWaitGroup.Wait()
+
+	if s.test.checkResult.Err != nil {
+		return fmt.Errorf("can not run checker in sandbox, error: %v", s.test.checkResult.Err)
+	}
+
+	switch s.test.checkResult.Verdict {
+	case verdict.OK, verdict.RT:
+		logger.Trace("Finished checker run for %s", s.loggerData)
+		return nil
+	case verdict.TL:
+		return fmt.Errorf("checker running took more than %v time", s.test.checkConfig.TimeLimit)
+	case verdict.ML:
+		return fmt.Errorf("checker running took more than %v memory", s.test.checkConfig.MemoryLimit)
+	case verdict.WL:
+		return fmt.Errorf("checker running took more than %v wall time", s.test.checkConfig.WallTimeLimit)
+	case verdict.SE:
+		return fmt.Errorf("checker security violation")
+	default:
+		return fmt.Errorf("unknown checker sandbox run verdict: %s", s.test.checkResult.Verdict)
+	}
+}
+
+func (s *JobPipelineState) runChecker() {
+	s.test.checkResult = s.sandbox.Run(s.test.checkConfig)
+	s.executeWaitGroup.Done()
+}
+
+func (s *JobPipelineState) parseCheckerResult() error {
+	_, err := os.Stat(filepath.Join(s.sandbox.Dir(), checkResultFile))
+	if err == nil {
+		return s.parseTestlibCheckerResult()
+	} else if errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("no checker result file found, non testlib checkers are not supported")
+	} else {
+		return fmt.Errorf("can not stat checker result file")
+	}
+}
+
+func (s *JobPipelineState) parseTestlibCheckerResult() error {
+	checkResultReader, err := os.Open(filepath.Join(s.sandbox.Dir(), checkResultFile))
+	if err != nil {
+		return fmt.Errorf(
+			"checker exited with exit code %d, can not parse checker result xml file in appes mode",
+			s.test.checkResult.Statistics.ExitCode,
+		)
+	}
+	defer checkResultReader.Close()
+
+	var checkerResult CheckerResultXML
+	xmlReader := xml.NewDecoder(checkResultReader)
+	xmlReader.CharsetReader = charset.NewReaderLabel
+	err = xmlReader.Decode(&checkerResult)
+	if err != nil {
+		return fmt.Errorf(
+			"can not parse checker result xml file in appes mode: %s",
+			err.Error(),
+		)
+	}
+
+	s.test.checkerOutputReader = s.limitedReader(strings.NewReader(checkerResult.Value))
+	switch checkerResult.Outcome {
+	case "accepted":
+		s.test.runResult.Verdict = verdict.OK
+	case "wrong-answer", "presentation-error", "unexpected-eof": // We will treat PE as WA as all testing systems now do
+		s.test.runResult.Verdict = verdict.WA
+	case "fail":
+		// Only in case of checker CF verdict we accept job as successful
+		s.test.runResult.Verdict = verdict.CF
+	case "points", "relative-scoring":
+		if checkerResult.Points == nil {
+			return fmt.Errorf(
+				"checker exited with exit code %d and verdict %s, but no points specified",
+				s.test.checkResult.Statistics.ExitCode,
+				checkerResult.Outcome,
+			)
+		} else {
+			s.test.runResult.Verdict = verdict.PT
+			s.test.runResult.Points = checkerResult.Points
+		}
+	default:
+		return fmt.Errorf(
+			"unknown checker verdict %s, checker exited with exit code %d",
+			checkerResult.Outcome,
+			s.test.checkResult.Statistics.ExitCode,
+		)
+	}
+
+	logger.Trace(
+		"Parsed testlib checker result for %s, checker verdict is %s",
+		s.loggerData, s.test.checkResult.Verdict,
+	)
+	return nil
+}
+
+func (s *JobPipelineState) uploadTestRunResources() error {
+	err := s.uploadOutput(testOutputFile, resource.TestOutput)
+	if err != nil {
+		return err
+	}
+
+	err = s.uploadOutput(testErrorFile, resource.TestStderr)
+	if err != nil {
+		return err
+	}
+
+	err = s.uploadCheckerOutput()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *JobPipelineState) uploadCheckerOutput() error {
+	checkerOutputRequest := &storageconn.Request{
+		Resource: resource.CheckerOutput,
+		SubmitID: uint64(s.job.Submission.ID),
+		TestID:   s.job.Test,
+		Files: map[string]io.Reader{
+			checkOutputFile: s.test.checkerOutputReader,
+		},
+	}
+	resp := s.invoker.TS.StorageConn.Upload(checkerOutputRequest)
+	if resp.Error != nil {
+		return fmt.Errorf("can not upload checker output to storage, error: %s", resp.Error.Error())
+	}
+	logger.Trace("Uploaded checker output for %s", s.loggerData)
+	return nil
+}
+
+type CheckerResultXML struct {
+	Outcome string   `xml:"outcome,attr"`
+	Points  *float64 `xml:"points,attr,omitempty"`
+	Value   string   `xml:",chardata"`
+}