From ba17768559a4ef84387fbf25026caea66f0112c8 Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Mon, 14 Apr 2025 23:48:56 +0300 Subject: [PATCH 01/10] WIP ioi scheduler --- common/db/models/problem.go | 77 +++++- master/queue/jobgenerators/generator.go | 4 +- master/queue/jobgenerators/icpc_generator.go | 4 +- .../jobgenerators/icpc_generator_test.go | 2 +- master/queue/jobgenerators/ioi_generator.go | 231 ++++++++++++++++++ .../queue/jobgenerators/ioi_generator_test.go | 1 + master/queue/queue_test.go | 10 +- 7 files changed, 315 insertions(+), 14 deletions(-) create mode 100644 master/queue/jobgenerators/ioi_generator.go create mode 100644 master/queue/jobgenerators/ioi_generator_test.go diff --git a/common/db/models/problem.go b/common/db/models/problem.go index 79031e8..4d9bbc9 100644 --- a/common/db/models/problem.go +++ b/common/db/models/problem.go @@ -1,16 +1,81 @@ package models import ( + "database/sql/driver" + "encoding/json" + "errors" "gorm.io/gorm" + "gorm.io/gorm/schema" "testing_system/lib/customfields" ) type ProblemType int +// TestGroupScoringType sets how should scheduler set points for a group +type TestGroupScoringType int + +// TestGroupFeedbackType sets which info about tests in a group would be shown +type TestGroupFeedbackType int + +const ( + ProblemTypeICPC ProblemType = iota + 1 + ProblemTypeIOI +) const ( - ProblemType_ICPC ProblemType = iota + 1 - ProblemType_IOI + // TestGroupScoringTypeComplete means that group costs TestGroup.Score (all the tests should be OK) + TestGroupScoringTypeComplete TestGroupScoringType = iota + 1 + // TestGroupScoringTypeEachTest means that group score = TestGroup.TestScore * (number of tests with OK) + TestGroupScoringTypeEachTest + // TestGroupScoringTypeMin means that group score = min(checker's scores among all the tests) + TestGroupScoringTypeMin ) +const ( + // TestGroupFeedbackTypeNone won't show anything + TestGroupFeedbackTypeNone TestGroupFeedbackType = iota + 1 + // TestGroupFeedbackTypePoints will show points only + TestGroupFeedbackTypePoints + // TestGroupFeedbackTypeICPC will show verdict, time and memory usage for the first test with no OK + TestGroupFeedbackTypeICPC + // TestGroupFeedbackTypeComplete same as TestGroupFeedbackTypeICPC, but for every test + TestGroupFeedbackTypeComplete + // TestGroupFeedbackTypeFull same as TestGroupFeedbackTypeComplete, but with input, output, stderr, etc. + TestGroupFeedbackTypeFull +) + +type TestGroup struct { + Name string `json:"name" yaml:"name"` + FirstTest int `json:"first_test" yaml:"first_test"` + LastTest int `json:"last_test" yaml:"last_test"` + // TestScore meaningful only in case of TestGroupScoringTypeEachTest + TestScore *float64 `json:"test_score" yaml:"test_score"` + // Score meaningful only in case of TestGroupScoringTypeComplete + Score *float64 `json:"score" yaml:"score"` + ScoringType TestGroupScoringType `json:"scoring_type" yaml:"scoring_type"` + FeedbackType TestGroupFeedbackType `json:"feedback_type" yaml:"feedback_type"` + RequiredGroups []string `json:"required_groups" yaml:"required_groups"` +} + +func (t TestGroup) Value() (driver.Value, error) { + return json.Marshal(t) +} + +func (t *TestGroup) Scan(value interface{}) error { + bytes, ok := value.([]byte) + if !ok { + return errors.New("type assertion to []byte failed") + } + return json.Unmarshal(bytes, t) +} + +func (t TestGroup) GormDBDataType(db *gorm.DB, field *schema.Field) string { + switch db.Dialector.Name() { + case "mysql", "sqlite": + return "JSON" + case "postgres": + return "JSONB" + } + return "" +} type Problem struct { gorm.Model @@ -21,17 +86,19 @@ type Problem struct { MemoryLimit customfields.Memory `yaml:"MemoryLimit"` TestsNumber uint64 `yaml:"TestsNumber"` + // TestGroups ignored for ICPC problems + TestGroups []TestGroup // WallTimeLimit specifies maximum execution and wait time. // By default, it is max(5s, TimeLimit * 2) WallTimeLimit *customfields.Time `yaml:"WallTimeLimit,omitempty"` - // MaxOpenFiles specifies maximum number of files, opened by testing system. + // MaxOpenFiles specifies the maximum number of files, opened by testing system. // By default, it is 64 MaxOpenFiles *uint64 `yaml:"MaxOpenFiles,omitempty"` - // MaxThreads specifies maximum number of threads and/or processes - // By default, it is single thread + // MaxThreads specifies the maximum number of threads and/or processes; + // By default, it is a single thread // If MaxThreads equals to -1, any number of threads allowed MaxThreads *int64 `yaml:"MaxThreads,omitempty"` diff --git a/master/queue/jobgenerators/generator.go b/master/queue/jobgenerators/generator.go index f2e6514..dbcda13 100644 --- a/master/queue/jobgenerators/generator.go +++ b/master/queue/jobgenerators/generator.go @@ -21,8 +21,10 @@ type Generator interface { func NewGenerator(problem *models.Problem, submission *models.Submission) (Generator, error) { switch problem.ProblemType { - case models.ProblemType_ICPC: + case models.ProblemTypeICPC: return newICPCGenerator(problem, submission) + case models.ProblemTypeIOI: + return NewIOIGenerator(problem, submission) default: return nil, fmt.Errorf("unknown problem type %v", problem.ProblemType) } diff --git a/master/queue/jobgenerators/icpc_generator.go b/master/queue/jobgenerators/icpc_generator.go index e99e759..b8c9fb8 100644 --- a/master/queue/jobgenerators/icpc_generator.go +++ b/master/queue/jobgenerators/icpc_generator.go @@ -130,7 +130,7 @@ func (i *ICPCGenerator) JobCompleted(result *masterconn.InvokerJobResult) (*mode defer i.mutex.Unlock() job, ok := i.givenJobs[result.JobID] if !ok { - return nil, fmt.Errorf("job not found") + return nil, fmt.Errorf("job %s not exist", result.JobID) } delete(i.givenJobs, result.JobID) @@ -150,7 +150,7 @@ func newICPCGenerator(problem *models.Problem, submission *models.Submission) (G logger.Panic("Can't generate generator id: %w", err) } - if problem.ProblemType != models.ProblemType_ICPC { + if problem.ProblemType != models.ProblemTypeICPC { return nil, fmt.Errorf("problem %v is not ICPC", problem.ID) } futureTests := make([]uint64, 0, problem.TestsNumber) diff --git a/master/queue/jobgenerators/icpc_generator_test.go b/master/queue/jobgenerators/icpc_generator_test.go index 4115e50..467e866 100644 --- a/master/queue/jobgenerators/icpc_generator_test.go +++ b/master/queue/jobgenerators/icpc_generator_test.go @@ -12,7 +12,7 @@ import ( func fixtureProblem() *models.Problem { return &models.Problem{ - ProblemType: models.ProblemType_ICPC, + ProblemType: models.ProblemTypeICPC, TestsNumber: 10, } } diff --git a/master/queue/jobgenerators/ioi_generator.go b/master/queue/jobgenerators/ioi_generator.go new file mode 100644 index 0000000..bfd3c00 --- /dev/null +++ b/master/queue/jobgenerators/ioi_generator.go @@ -0,0 +1,231 @@ +package jobgenerators + +import ( + "fmt" + "github.com/google/uuid" + "slices" + "sync" + "testing_system/common/connectors/invokerconn" + "testing_system/common/connectors/masterconn" + "testing_system/common/constants/verdict" + "testing_system/common/db/models" + "testing_system/lib/logger" +) + +type IOIGenerator struct { + id string + mutex sync.Mutex + submission *models.Submission + problem *models.Problem + state state + givenJobs map[string]*invokerconn.Job + + groupNameToGroup map[string]*models.TestGroup + groupNamesToBeGiven []string + groupNameToDependencies map[string][]*models.TestGroup +} + +func (i *IOIGenerator) finalizeResults() { + for _, group := range i.groupNameToGroup { + switch group.ScoringType { + case models.TestGroupScoringTypeComplete: + setSkipped := false + for j := range i.submission.TestResults[group.FirstTest-1 : group.LastTest] { + if setSkipped { + i.submission.TestResults[j].Verdict = verdict.SK + continue + } + if i.submission.TestResults[j].Verdict == verdict.OK || i.submission.TestResults[j].Verdict == verdict.SK { + continue + } + setSkipped = true + i.submission.Verdict = i.submission.TestResults[j].Verdict + } + if !setSkipped { + if group.Score == nil { + logger.Panic("Group '%v' in problemId=%v has TypeComplete, but score is nil", + group.Name, i.problem.ID) + } + i.submission.TestResults[group.LastTest-1].Points = group.Score + } + case models.TestGroupScoringTypeEachTest: + // TODO + case models.TestGroupScoringTypeMin: + // TODO + } + } + notOkInd := slices.IndexFunc(i.submission.TestResults, func(result models.TestResult) bool { + return result.Verdict != verdict.OK + }) + if notOkInd == -1 { + i.submission.Verdict = verdict.OK + } else { + i.submission.Verdict = i.submission.TestResults[notOkInd].Verdict + } + totalScore := 0. + for _, result := range i.submission.TestResults { + if result.Points != nil { + totalScore += *result.Points + } + } + i.submission.Score = totalScore +} + +func (i *IOIGenerator) fetchGroupDependencies(curGroup *models.TestGroup) map[string]struct{} { + used := make(map[string]struct{}) + q := make([]*models.TestGroup, 0) + q = append(q, curGroup) + used[curGroup.Name] = struct{}{} + for len(q) > 0 { + group := q[0] + q = q[1:] + for _, nextGroup := range i.groupNameToDependencies[group.Name] { + if _, ok := used[nextGroup.Name]; !ok { + q = append(q, nextGroup) + used[nextGroup.Name] = struct{}{} + } + } + } + return used +} + +func (i *IOIGenerator) testNumberToGroup(testNumber uint64) *models.TestGroup { + for _, group := range i.groupNameToGroup { + if group.FirstTest <= int(testNumber) && int(testNumber) <= group.LastTest { + return group + } + } + return nil +} + +func (i *IOIGenerator) ID() string { + return i.id +} + +func (i *IOIGenerator) NextJob() *invokerconn.Job { + i.mutex.Lock() + defer i.mutex.Unlock() + if i.state == compilationFinished && len(i.groupNamesToBeGiven) == 0 { + return nil + } + if i.state == compilationStarted { + return nil + } + id, err := uuid.NewV7() + if err != nil { + logger.Panic("Can't generate id for job: %w", err) + } + job := &invokerconn.Job{ + ID: id.String(), + SubmitID: i.submission.ID, + } + if i.state == compilationNotStarted { + job.Type = invokerconn.CompileJob + i.state = compilationStarted + return job + } + curGroupName := i.groupNamesToBeGiven[0] + group := i.groupNameToGroup[curGroupName] + job.Type = invokerconn.TestJob + job.Test = uint64(group.FirstTest) + if group.FirstTest == group.LastTest { + i.groupNamesToBeGiven = i.groupNamesToBeGiven[1:] + } else { + group.FirstTest++ + } + i.givenJobs[job.ID] = job + return job +} + +// compileJobCompleted must be done with acquired mutex +func (i *IOIGenerator) compileJobCompleted(job *invokerconn.Job, result *masterconn.InvokerJobResult) (*models.Submission, error) { + if job.Type != invokerconn.CompileJob { + return nil, fmt.Errorf("job type %s is not compile job", job.ID) + } + switch result.Verdict { + case verdict.CD: + i.state = compilationFinished + return nil, nil + case verdict.CE: + i.submission.Verdict = result.Verdict + return i.submission, nil + default: + return nil, fmt.Errorf("unknown verdict for compilation completed: %v", result.Verdict) + } +} + +// testJobCompleted must be done with acquired mutex +func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn.InvokerJobResult) (*models.Submission, error) { + if job.Type != invokerconn.TestJob { + return nil, fmt.Errorf("job type %s is not test job", job.ID) + } + group := i.testNumberToGroup(job.Test) + if group == nil { + logger.Panic("Can't get group of test %v in job %v, problemId=%v", job.Test, job.ID, i.problem.ID) + return nil, nil // just for goland to chill + } + + i.submission.TestResults[job.Test-1].Verdict = result.Verdict + + switch result.Verdict { + case verdict.OK: + if group.ScoringType == models.TestGroupScoringTypeEachTest { + if group.TestScore == nil { + logger.Panic("Group '%v' has type EachTest, but TestScore is nil in problemId=%v", + group.Name, i.problem.ID) + } + i.submission.TestResults[job.Test-1].Points = group.TestScore + } else if group.ScoringType == models.TestGroupScoringTypeMin { + if result.Points == nil { + logger.Panic("Group '%v' has type Min, but checker didn't set points in problemId=%v, jobId=%v", + group.Name, i.problem.ID, job.ID) + } + i.submission.TestResults[job.Test-1].Points = result.Points + } + if len(i.givenJobs) == 0 && len(i.groupNamesToBeGiven) == 0 { + i.finalizeResults() + return i.submission, nil + } + return nil, nil + case verdict.PT, verdict.WA, verdict.RT, verdict.ML, verdict.TL, verdict.WL, verdict.SE, verdict.CF: + dependencies := i.fetchGroupDependencies(group) + newGroupNamesToBeGiven := make([]string, 0) + for _, s := range i.groupNamesToBeGiven { + if _, ok := dependencies[s]; !ok { + newGroupNamesToBeGiven = append(newGroupNamesToBeGiven, s) + } else if s == group.Name && group.ScoringType != models.TestGroupScoringTypeComplete { + newGroupNamesToBeGiven = append(newGroupNamesToBeGiven, s) + } + } + i.groupNamesToBeGiven = newGroupNamesToBeGiven + if len(i.givenJobs) == 0 && len(i.groupNamesToBeGiven) == 0 { + i.finalizeResults() + return i.submission, nil + } + return nil, nil + default: + return nil, fmt.Errorf("unknown verdict for testing completed: %v", result.Verdict) + } +} + +func (i *IOIGenerator) JobCompleted(jobResult *masterconn.InvokerJobResult) (*models.Submission, error) { + i.mutex.Lock() + defer i.mutex.Unlock() + job, ok := i.givenJobs[jobResult.JobID] + if !ok { + return nil, fmt.Errorf("job %s not exist", jobResult.JobID) + } + delete(i.givenJobs, jobResult.JobID) + switch job.Type { + case invokerconn.CompileJob: + return i.compileJobCompleted(job, jobResult) + case invokerconn.TestJob: + return i.testJobCompleted(job, jobResult) + default: + return nil, fmt.Errorf("unknown job type for IOI problem: %v", job.Type) + } +} + +func NewIOIGenerator(problem *models.Problem, submission *models.Submission) (Generator, error) { + return &IOIGenerator{}, nil +} diff --git a/master/queue/jobgenerators/ioi_generator_test.go b/master/queue/jobgenerators/ioi_generator_test.go new file mode 100644 index 0000000..1601935 --- /dev/null +++ b/master/queue/jobgenerators/ioi_generator_test.go @@ -0,0 +1 @@ +package jobgenerators diff --git a/master/queue/queue_test.go b/master/queue/queue_test.go index 9d3e3ef..534e2a2 100644 --- a/master/queue/queue_test.go +++ b/master/queue/queue_test.go @@ -54,11 +54,11 @@ func TestQueueWork(t *testing.T) { q := NewQueue(nil).(*Queue) problem1 := models.Problem{ TestsNumber: 2, - ProblemType: models.ProblemType_ICPC, + ProblemType: models.ProblemTypeICPC, } problem2 := models.Problem{ TestsNumber: 2, - ProblemType: models.ProblemType_ICPC, + ProblemType: models.ProblemTypeICPC, } problem1.ID, problem2.ID = 1, 2 submission1 := models.Submission{} @@ -77,11 +77,11 @@ func TestQueueFairness(t *testing.T) { q := NewQueue(nil).(*Queue) problem1 := models.Problem{ TestsNumber: 500, - ProblemType: models.ProblemType_ICPC, + ProblemType: models.ProblemTypeICPC, } problem2 := models.Problem{ TestsNumber: 10, - ProblemType: models.ProblemType_ICPC, + ProblemType: models.ProblemTypeICPC, } problem1.ID, problem2.ID = 1, 2 submission1 := models.Submission{} @@ -104,7 +104,7 @@ func TestQueue_RescheduleJob(t *testing.T) { require.True(t, isQueueEmpty(q)) problem1 := models.Problem{ TestsNumber: 1, - ProblemType: models.ProblemType_ICPC, + ProblemType: models.ProblemTypeICPC, } problem2 := problem1 submission1 := models.Submission{} From 4ba4055ec5b829f346ccd441d68f66302a97de79 Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Thu, 17 Apr 2025 01:11:16 +0300 Subject: [PATCH 02/10] fixes and tests --- common/db/models/problem.go | 18 +- common/db/models/submission.go | 33 + master/queue/jobgenerators/common.go | 10 + master/queue/jobgenerators/generator_test.go | 714 ++++++++++++++++++ master/queue/jobgenerators/icpc_generator.go | 11 +- .../jobgenerators/icpc_generator_test.go | 254 ------- master/queue/jobgenerators/ioi_generator.go | 377 ++++++--- .../queue/jobgenerators/ioi_generator_test.go | 1 - 8 files changed, 1060 insertions(+), 358 deletions(-) create mode 100644 master/queue/jobgenerators/common.go create mode 100644 master/queue/jobgenerators/generator_test.go delete mode 100644 master/queue/jobgenerators/icpc_generator_test.go delete mode 100644 master/queue/jobgenerators/ioi_generator_test.go diff --git a/common/db/models/problem.go b/common/db/models/problem.go index 4d9bbc9..954f842 100644 --- a/common/db/models/problem.go +++ b/common/db/models/problem.go @@ -49,10 +49,10 @@ type TestGroup struct { // TestScore meaningful only in case of TestGroupScoringTypeEachTest TestScore *float64 `json:"test_score" yaml:"test_score"` // Score meaningful only in case of TestGroupScoringTypeComplete - Score *float64 `json:"score" yaml:"score"` - ScoringType TestGroupScoringType `json:"scoring_type" yaml:"scoring_type"` - FeedbackType TestGroupFeedbackType `json:"feedback_type" yaml:"feedback_type"` - RequiredGroups []string `json:"required_groups" yaml:"required_groups"` + Score *float64 `json:"score" yaml:"score"` + ScoringType TestGroupScoringType `json:"scoring_type" yaml:"scoring_type"` + FeedbackType TestGroupFeedbackType `json:"feedback_type" yaml:"feedback_type"` + RequiredGroupNames []string `json:"required_groups" yaml:"required_groups"` } func (t TestGroup) Value() (driver.Value, error) { @@ -80,6 +80,7 @@ func (t TestGroup) GormDBDataType(db *gorm.DB, field *schema.Field) string { type Problem struct { gorm.Model +<<<<<<< HEAD ProblemType ProblemType `yaml:"ProblemType"` TimeLimit customfields.Time `yaml:"TimeLimit"` @@ -88,14 +89,23 @@ type Problem struct { TestsNumber uint64 `yaml:"TestsNumber"` // TestGroups ignored for ICPC problems TestGroups []TestGroup +======= + ProblemType ProblemType +>>>>>>> f1bf2b0 (fixes and tests) // WallTimeLimit specifies maximum execution and wait time. // By default, it is max(5s, TimeLimit * 2) WallTimeLimit *customfields.Time `yaml:"WallTimeLimit,omitempty"` +<<<<<<< HEAD // MaxOpenFiles specifies the maximum number of files, opened by testing system. // By default, it is 64 MaxOpenFiles *uint64 `yaml:"MaxOpenFiles,omitempty"` +======= + TestsNumber uint64 + // TestGroups ignored for ICPC problems + TestGroups []TestGroup +>>>>>>> f1bf2b0 (fixes and tests) // MaxThreads specifies the maximum number of threads and/or processes; // By default, it is a single thread diff --git a/common/db/models/submission.go b/common/db/models/submission.go index c0182b2..77eef27 100644 --- a/common/db/models/submission.go +++ b/common/db/models/submission.go @@ -16,6 +16,7 @@ type TestResult struct { Verdict verdict.Verdict `json:"Verdict" yaml:"Verdict"` Time customfields.Time `json:"Time" yaml:"Time"` Memory customfields.Memory `json:"Memory" yaml:"Memory"` + Error string `json:"Error,omitempty" yaml:"Error,omitempty"` } type TestResults []TestResult @@ -42,6 +43,37 @@ func (t TestResults) GormDBDataType(db *gorm.DB, field *schema.Field) string { return "" } +type GroupResult struct { + GroupName string `json:"GroupName" yaml:"GroupName"` + Points float64 `json:"Points" yaml:"Points"` + Passed bool `json:"Passed" yaml:"Passed"` + // TODO maybe more fields +} + +type GroupResults []GroupResult + +func (t GroupResults) Value() (driver.Value, error) { + return json.Marshal(t) +} + +func (t *GroupResults) Scan(value interface{}) error { + bytes, ok := value.([]byte) + if !ok { + return errors.New("type assertion to []byte failed") + } + return json.Unmarshal(bytes, t) +} + +func (t GroupResults) GormDBDataType(db *gorm.DB, field *schema.Field) string { + switch db.Dialector.Name() { + case "mysql", "sqlite": + return "JSON" + case "postgres": + return "JSONB" + } + return "" +} + type Submission struct { gorm.Model ProblemID uint64 `json:"ProblemID" yaml:"ProblemID"` @@ -50,4 +82,5 @@ type Submission struct { Score float64 `json:"Score" yaml:"Score"` Verdict verdict.Verdict `json:"Verdict" yaml:"Verdict"` TestResults TestResults `gorm:"type:jsonb" json:"TestResults" yaml:"TestResults"` + GroupResults GroupResults } diff --git a/master/queue/jobgenerators/common.go b/master/queue/jobgenerators/common.go new file mode 100644 index 0000000..ba7ecbf --- /dev/null +++ b/master/queue/jobgenerators/common.go @@ -0,0 +1,10 @@ +package jobgenerators + +// used in IOI and ICPC generators +type generatorState int + +const ( + compilationNotStarted generatorState = iota + compilationStarted + compilationFinished +) diff --git a/master/queue/jobgenerators/generator_test.go b/master/queue/jobgenerators/generator_test.go new file mode 100644 index 0000000..1361e86 --- /dev/null +++ b/master/queue/jobgenerators/generator_test.go @@ -0,0 +1,714 @@ +package jobgenerators + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/xorcare/pointer" + "testing" + "testing_system/common/connectors/invokerconn" + "testing_system/common/connectors/masterconn" + "testing_system/common/constants/verdict" + "testing_system/common/db/models" +) + +func fixtureICPCProblem() *models.Problem { + return &models.Problem{ + ProblemType: models.ProblemTypeICPC, + TestsNumber: 10, + } +} + +func fixtureSubmission() *models.Submission { + submission := &models.Submission{} + submission.ID = 1 + return submission +} + +func nextJob(t *testing.T, g Generator, SubmitID uint, jobType invokerconn.JobType, test uint64) *invokerconn.Job { + job := g.NextJob() + assert.NotNil(t, job) + assert.Equal(t, job.Type, jobType) + assert.Equal(t, SubmitID, job.SubmitID) + assert.Equal(t, test, job.Test) + return job +} + +func noJobs(t *testing.T, g Generator) { + job := g.NextJob() + assert.Nil(t, job) +} + +func TestICPCGenerator(t *testing.T) { + t.Run("Fail compilation", func(t *testing.T) { + problem, submission := fixtureICPCProblem(), fixtureSubmission() + g, err := NewGenerator(problem, submission) + require.Nil(t, err) + job := nextJob(t, g, 1, invokerconn.CompileJob, 0) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CE, + }) + require.NotNil(t, sub) + require.Nil(t, err) + + require.Equal(t, verdict.CE, sub.Verdict) + require.Equal(t, 0., sub.Score) + for i, result := range sub.TestResults { + require.Equal(t, verdict.SK, result.Verdict) + require.Equal(t, uint64(i)+1, result.TestNumber) + } + }) + + t.Run("Straight tasks finishing", func(t *testing.T) { + problem := fixtureICPCProblem() + submission := fixtureSubmission() + generator, err := NewGenerator(problem, submission) + require.Nil(t, err) + job := nextJob(t, generator, 1, invokerconn.CompileJob, 0) + noJobs(t, generator) + sub, err := generator.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.Nil(t, sub) + require.Nil(t, err) + for i := range 9 { + job = nextJob(t, generator, 1, invokerconn.TestJob, uint64(i)+1) + sub, err = generator.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + require.Nil(t, sub) + require.Nil(t, err) + } + job = nextJob(t, generator, 1, invokerconn.TestJob, 10) + sub, err = generator.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + require.NotNil(t, sub) + require.Nil(t, err) + + require.Equal(t, verdict.OK, sub.Verdict) + require.Equal(t, 1., sub.Score) + for i, result := range sub.TestResults { + require.Equal(t, verdict.OK, result.Verdict) + require.Equal(t, uint64(i)+1, result.TestNumber) + } + }) + + t.Run("Tasks finishing", func(t *testing.T) { + prepare := func() (Generator, []string) { + problem := fixtureICPCProblem() + submission := fixtureSubmission() + g, err := NewGenerator(problem, submission) + require.Nil(t, err) + job := nextJob(t, g, 1, invokerconn.CompileJob, 0) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.Nil(t, sub) + require.Nil(t, err) + firstTwoJobIDs := make([]string, 0) + for i := range 2 { + job = nextJob(t, g, 1, invokerconn.TestJob, uint64(i)+1) + firstTwoJobIDs = append(firstTwoJobIDs, job.ID) + } + return g, firstTwoJobIDs + } + finishOtherTests := func(g Generator) { + for i := 2; i < 9; i++ { + job := nextJob(t, g, 1, invokerconn.TestJob, uint64(i)+1) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + require.Nil(t, sub) + require.Nil(t, err) + } + job := nextJob(t, g, 1, invokerconn.TestJob, 10) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + require.NotNil(t, sub) + require.Nil(t, err) + + require.Equal(t, verdict.OK, sub.Verdict) + require.Equal(t, 1., sub.Score) + for i, result := range sub.TestResults { + require.Equal(t, verdict.OK, result.Verdict) + require.Equal(t, uint64(i)+1, result.TestNumber) + } + } + + t.Run("right order", func(t *testing.T) { + g, firstTwoJobIDs := prepare() + for _, id := range firstTwoJobIDs { + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: id, + Verdict: verdict.OK, + }) + require.Nil(t, sub) + require.Nil(t, err) + } + finishOtherTests(g) + }) + + t.Run("wrong order + both ok", func(t *testing.T) { + g, firstTwoJobIDs := prepare() + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: firstTwoJobIDs[1], + Verdict: verdict.OK, + }) + require.Nil(t, sub) + require.Nil(t, err) + + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: firstTwoJobIDs[0], + Verdict: verdict.OK, + }) + require.Nil(t, sub) + require.Nil(t, err) + + finishOtherTests(g) + }) + + t.Run("wrong order + 2nd fail", func(t *testing.T) { + g, firstTwoJobIDs := prepare() + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: firstTwoJobIDs[1], + Verdict: verdict.WA, + }) + require.Nil(t, sub) + require.Nil(t, err) + + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: firstTwoJobIDs[0], + Verdict: verdict.OK, + }) + require.NotNil(t, sub) + require.Nil(t, err) + + require.Equal(t, verdict.WA, sub.Verdict) + require.Equal(t, 0., sub.Score) + require.Equal(t, verdict.OK, sub.TestResults[0].Verdict) + require.Equal(t, verdict.WA, sub.TestResults[1].Verdict) + for i, result := range sub.TestResults[2:] { + require.Equal(t, verdict.SK, result.Verdict) + require.Equal(t, uint64(i)+3, result.TestNumber) + } + }) + + t.Run("wrong order + 1st fail", func(t *testing.T) { + g, firstTwoJobIDs := prepare() + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: firstTwoJobIDs[1], + Verdict: verdict.OK, + }) + require.Nil(t, sub) + require.Nil(t, err) + + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: firstTwoJobIDs[0], + Verdict: verdict.WA, + }) + require.NotNil(t, sub) + require.Nil(t, err) + + require.Equal(t, verdict.WA, sub.Verdict) + require.Equal(t, 0., sub.Score) + require.Equal(t, verdict.WA, sub.TestResults[0].Verdict) + require.Equal(t, verdict.SK, sub.TestResults[1].Verdict) + for i, result := range sub.TestResults[2:] { + require.Equal(t, verdict.SK, result.Verdict) + require.Equal(t, uint64(i)+3, result.TestNumber) + } + }) + }) + + t.Run("Finish same job twice", func(t *testing.T) { + problem, submission := fixtureICPCProblem(), fixtureSubmission() + g, err := NewGenerator(problem, submission) + require.Nil(t, err) + job := nextJob(t, g, 1, invokerconn.CompileJob, 0) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CE, + }) + require.NotNil(t, sub) + require.Nil(t, err) + + require.Equal(t, verdict.CE, sub.Verdict) + require.Equal(t, 0., sub.Score) + for i, result := range sub.TestResults { + require.Equal(t, verdict.SK, result.Verdict) + require.Equal(t, uint64(i)+1, result.TestNumber) + } + + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CE, + }) + require.Nil(t, sub) + require.NotNil(t, err) + }) +} + +func TestIOIGenerator(t *testing.T) { + t.Run("Bad problem configuration", func(t *testing.T) { + badProblems := []models.Problem{ + // type EachTest, but TestScore is nil + { + ProblemType: models.ProblemTypeIOI, + TestGroups: []models.TestGroup{ + { + Name: "name", + FirstTest: 1, + LastTest: 1, + TestScore: nil, + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + }, + TestsNumber: 1, + }, + // type Complete, but Score is nil + { + ProblemType: models.ProblemTypeIOI, + TestGroups: []models.TestGroup{ + { + Name: "name", + FirstTest: 1, + LastTest: 1, + Score: nil, + ScoringType: models.TestGroupScoringTypeComplete, + RequiredGroupNames: make([]string, 0), + }, + }, + TestsNumber: 1, + }, + // cyclic groups + { + ProblemType: models.ProblemTypeIOI, + TestGroups: []models.TestGroup{ + { + Name: "name1", + FirstTest: 1, + LastTest: 1, + ScoringType: models.TestGroupScoringTypeMin, + RequiredGroupNames: []string{"name2"}, + }, + { + Name: "name2", + FirstTest: 2, + LastTest: 2, + ScoringType: models.TestGroupScoringTypeMin, + RequiredGroupNames: []string{"name1"}, + }, + }, + }, + // test not covered + { + ProblemType: models.ProblemTypeIOI, + TestGroups: []models.TestGroup{ + { + Name: "name", + FirstTest: 1, + LastTest: 1, + TestScore: nil, + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + }, + TestsNumber: 2, + }, + // test in several groups + { + ProblemType: models.ProblemTypeIOI, + TestGroups: []models.TestGroup{ + { + Name: "name", + FirstTest: 1, + LastTest: 2, + TestScore: nil, + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + { + Name: "name1", + FirstTest: 2, + LastTest: 3, + TestScore: nil, + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + }, + TestsNumber: 3, + }, + // the same group name + { + ProblemType: models.ProblemTypeIOI, + TestGroups: []models.TestGroup{ + { + Name: "name", + FirstTest: 1, + LastTest: 2, + TestScore: nil, + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + { + Name: "name", + FirstTest: 3, + LastTest: 3, + TestScore: nil, + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + }, + TestsNumber: 3, + }, + } + for _, problem := range badProblems { + _, err := NewIOIGenerator(&problem, &models.Submission{}) + require.Error(t, err) + } + }) + + problemWithOneGroup := models.Problem{ + ProblemType: models.ProblemTypeIOI, + TestsNumber: 10, + TestGroups: []models.TestGroup{ + { + Name: "group1", + FirstTest: 1, + LastTest: 10, + TestScore: nil, + Score: pointer.Float64(100), + ScoringType: models.TestGroupScoringTypeComplete, + RequiredGroupNames: make([]string, 0), + }, + }, + } + + t.Run("Fail compilation", func(t *testing.T) { + submission := fixtureSubmission() + wasProblem := problemWithOneGroup + g, err := NewGenerator(&problemWithOneGroup, submission) + require.Nil(t, err) + job := nextJob(t, g, 1, invokerconn.CompileJob, 0) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CE, + }) + require.NotNil(t, sub) + require.Nil(t, err) + + require.Equal(t, verdict.CE, sub.Verdict) + require.Equal(t, 0., sub.Score) + for i, result := range sub.TestResults { + require.Equal(t, verdict.SK, result.Verdict) + require.Equal(t, uint64(i)+1, result.TestNumber) + } + require.Equal(t, wasProblem, problemWithOneGroup) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 0., + Passed: false, + }, + }) + }) + + t.Run("Straight task finishing", func(t *testing.T) { + wasProblem := problemWithOneGroup + submission := fixtureSubmission() + generator, err := NewGenerator(&problemWithOneGroup, submission) + require.Nil(t, err) + job := nextJob(t, generator, 1, invokerconn.CompileJob, 0) + noJobs(t, generator) + sub, err := generator.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.Nil(t, sub) + require.Nil(t, err) + for i := range 9 { + job = nextJob(t, generator, 1, invokerconn.TestJob, uint64(i)+1) + sub, err = generator.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + require.Nil(t, sub) + require.Nil(t, err) + } + job = nextJob(t, generator, 1, invokerconn.TestJob, 10) + sub, err = generator.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + require.NotNil(t, sub) + require.Nil(t, err) + + require.Equal(t, verdict.OK, sub.Verdict) + require.Equal(t, 100., sub.Score) + for i, result := range sub.TestResults { + require.Equal(t, verdict.OK, result.Verdict) + require.Equal(t, uint64(i)+1, result.TestNumber) + } + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 100., + Passed: true, + }, + }) + + require.Equal(t, wasProblem, problemWithOneGroup) + }) + + t.Run("Fails in TestGroupScoringTypeComplete", func(t *testing.T) { + prepare := func() Generator { + problem := models.Problem{ + ProblemType: models.ProblemTypeIOI, + TestsNumber: 2, + TestGroups: []models.TestGroup{ + { + Name: "group1", + FirstTest: 1, + LastTest: 1, + TestScore: nil, + Score: pointer.Float64(50), + ScoringType: models.TestGroupScoringTypeComplete, + RequiredGroupNames: make([]string, 0), + }, + { + Name: "group2", + FirstTest: 2, + LastTest: 2, + TestScore: nil, + Score: pointer.Float64(50), + ScoringType: models.TestGroupScoringTypeComplete, + RequiredGroupNames: []string{"group1"}, + }, + }, + } + gen, err := NewGenerator(&problem, &models.Submission{}) + require.NoError(t, err) + job := nextJob(t, gen, 0, invokerconn.CompileJob, 0) + sub, err := gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.Nil(t, sub) + require.NoError(t, err) + return gen + } + + t.Run("WA 1st, OK 2nd", func(t *testing.T) { + gen := prepare() + job1 := nextJob(t, gen, 0, invokerconn.TestJob, 1) + job2 := nextJob(t, gen, 0, invokerconn.TestJob, 2) + sub, err := gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job1.ID, + Verdict: verdict.WA, + }) + require.Nil(t, sub) + require.NoError(t, err) + sub, err = gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job2.ID, + Verdict: verdict.OK, + }) + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 0., sub.Score) + require.Equal(t, verdict.WA, sub.TestResults[0].Verdict) + require.Equal(t, verdict.SK, sub.TestResults[1].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 0., + Passed: false, + }, + { + GroupName: "group2", + Points: 0., + Passed: false, + }, + }) + }) + + t.Run("OK 2nd, WA 1st", func(t *testing.T) { + gen := prepare() + job1 := nextJob(t, gen, 0, invokerconn.TestJob, 1) + job2 := nextJob(t, gen, 0, invokerconn.TestJob, 2) + sub, err := gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job2.ID, + Verdict: verdict.OK, + }) + require.Nil(t, sub) + require.NoError(t, err) + sub, err = gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job1.ID, + Verdict: verdict.WA, + }) + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 0., sub.Score) + require.Equal(t, verdict.WA, sub.TestResults[0].Verdict) + require.Equal(t, verdict.SK, sub.TestResults[1].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 0., + Passed: false, + }, + { + GroupName: "group2", + Points: 0., + Passed: false, + }, + }) + }) + + t.Run("OK 1st, WA 2nd", func(t *testing.T) { + gen := prepare() + job1 := nextJob(t, gen, 0, invokerconn.TestJob, 1) + job2 := nextJob(t, gen, 0, invokerconn.TestJob, 2) + sub, err := gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job1.ID, + Verdict: verdict.OK, + }) + require.Nil(t, sub) + require.NoError(t, err) + sub, err = gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job2.ID, + Verdict: verdict.WA, + }) + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 50., sub.Score) + require.Equal(t, verdict.OK, sub.TestResults[0].Verdict) + require.Equal(t, verdict.WA, sub.TestResults[1].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 50., + Passed: true, + }, + { + GroupName: "group2", + Points: 0., + Passed: false, + }, + }) + }) + + t.Run("WA 2nd, OK 1st", func(t *testing.T) { + gen := prepare() + job1 := nextJob(t, gen, 0, invokerconn.TestJob, 1) + job2 := nextJob(t, gen, 0, invokerconn.TestJob, 2) + sub, err := gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job2.ID, + Verdict: verdict.WA, + }) + require.Nil(t, sub) + require.NoError(t, err) + sub, err = gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job1.ID, + Verdict: verdict.OK, + }) + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 50., sub.Score) + require.Equal(t, verdict.OK, sub.TestResults[0].Verdict) + require.Equal(t, verdict.WA, sub.TestResults[1].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 50., + Passed: true, + }, + { + GroupName: "group2", + Points: 0., + Passed: false, + }, + }) + }) + + t.Run("WA 1st, then take 2nd", func(t *testing.T) { + gen := prepare() + job := nextJob(t, gen, 0, invokerconn.TestJob, 1) + sub, err := gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.WA, + }) + require.NotNil(t, sub) + require.NoError(t, err) + assert.Nil(t, gen.NextJob()) + + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 0., sub.Score) + require.Equal(t, verdict.WA, sub.TestResults[0].Verdict) + require.Equal(t, verdict.SK, sub.TestResults[1].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 0., + Passed: false, + }, + { + GroupName: "group2", + Points: 0., + Passed: false, + }, + }) + }) + + t.Run("Fail in group with >1 tests", func(t *testing.T) { + problem := models.Problem{ + ProblemType: models.ProblemTypeIOI, + TestsNumber: 2, + TestGroups: []models.TestGroup{ + { + Name: "group1", + FirstTest: 1, + LastTest: 2, + TestScore: nil, + Score: pointer.Float64(100), + ScoringType: models.TestGroupScoringTypeComplete, + RequiredGroupNames: make([]string, 0), + }, + }, + } + gen, err := NewGenerator(&problem, &models.Submission{}) + require.NoError(t, err) + job := nextJob(t, gen, 0, invokerconn.CompileJob, 0) + sub, err := gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.Nil(t, sub) + require.NoError(t, err) + job = nextJob(t, gen, 0, invokerconn.TestJob, 1) + sub, err = gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.WA, + }) + require.Nil(t, gen.NextJob()) + require.NotNil(t, sub) + require.NoError(t, err) + + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 0., sub.Score) + require.Equal(t, verdict.WA, sub.TestResults[0].Verdict) + require.Equal(t, verdict.SK, sub.TestResults[1].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 0., + Passed: false, + }, + }) + }) + }) +} diff --git a/master/queue/jobgenerators/icpc_generator.go b/master/queue/jobgenerators/icpc_generator.go index b8c9fb8..462df50 100644 --- a/master/queue/jobgenerators/icpc_generator.go +++ b/master/queue/jobgenerators/icpc_generator.go @@ -11,20 +11,12 @@ import ( "testing_system/lib/logger" ) -type state int - -const ( - compilationNotStarted state = iota - compilationStarted - compilationFinished -) - type ICPCGenerator struct { id string mutex sync.Mutex submission *models.Submission problem *models.Problem - state state + state generatorState hasFails bool givenJobs map[string]*invokerconn.Job futureTests []uint64 @@ -170,6 +162,7 @@ func newICPCGenerator(problem *models.Problem, submission *models.Submission) (G return &ICPCGenerator{ id: id.String(), submission: submission, + state: compilationNotStarted, problem: problem, givenJobs: make(map[string]*invokerconn.Job), futureTests: futureTests, diff --git a/master/queue/jobgenerators/icpc_generator_test.go b/master/queue/jobgenerators/icpc_generator_test.go deleted file mode 100644 index 467e866..0000000 --- a/master/queue/jobgenerators/icpc_generator_test.go +++ /dev/null @@ -1,254 +0,0 @@ -package jobgenerators - -import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "testing" - "testing_system/common/connectors/invokerconn" - "testing_system/common/connectors/masterconn" - "testing_system/common/constants/verdict" - "testing_system/common/db/models" -) - -func fixtureProblem() *models.Problem { - return &models.Problem{ - ProblemType: models.ProblemTypeICPC, - TestsNumber: 10, - } -} - -func fixtureSubmission() *models.Submission { - submission := &models.Submission{} - submission.ID = 1 - return submission -} - -func nextJob(t *testing.T, g Generator, SubmitID uint, jobType invokerconn.JobType, test uint64) *invokerconn.Job { - job := g.NextJob() - assert.NotNil(t, job) - assert.Equal(t, job.Type, jobType) - assert.Equal(t, SubmitID, job.SubmitID) - assert.Equal(t, test, job.Test) - return job -} - -func noJobs(t *testing.T, g Generator) { - job := g.NextJob() - assert.Nil(t, job) -} - -func TestStraightTasksFinishing(t *testing.T) { - problem := fixtureProblem() - submission := fixtureSubmission() - generator, err := NewGenerator(problem, submission) - require.Nil(t, err) - job := nextJob(t, generator, 1, invokerconn.CompileJob, 0) - noJobs(t, generator) - sub, err := generator.JobCompleted(&masterconn.InvokerJobResult{ - JobID: job.ID, - Verdict: verdict.CD, - }) - require.Nil(t, sub) - require.Nil(t, err) - for i := range 9 { - job = nextJob(t, generator, 1, invokerconn.TestJob, uint64(i)+1) - sub, err = generator.JobCompleted(&masterconn.InvokerJobResult{ - JobID: job.ID, - Verdict: verdict.OK, - }) - require.Nil(t, sub) - require.Nil(t, err) - } - job = nextJob(t, generator, 1, invokerconn.TestJob, 10) - sub, err = generator.JobCompleted(&masterconn.InvokerJobResult{ - JobID: job.ID, - Verdict: verdict.OK, - }) - require.NotNil(t, sub) - require.Nil(t, err) - - require.Equal(t, verdict.OK, sub.Verdict) - require.Equal(t, 1., sub.Score) - for i, result := range sub.TestResults { - require.Equal(t, verdict.OK, result.Verdict) - require.Equal(t, uint64(i)+1, result.TestNumber) - } -} - -func TestTasksFinishing(t *testing.T) { - prepare := func() (Generator, []string) { - problem := fixtureProblem() - submission := fixtureSubmission() - g, err := NewGenerator(problem, submission) - require.Nil(t, err) - job := nextJob(t, g, 1, invokerconn.CompileJob, 0) - sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: job.ID, - Verdict: verdict.CD, - }) - require.Nil(t, sub) - require.Nil(t, err) - firstTwoJobIDs := make([]string, 0) - for i := range 2 { - job = nextJob(t, g, 1, invokerconn.TestJob, uint64(i)+1) - firstTwoJobIDs = append(firstTwoJobIDs, job.ID) - } - return g, firstTwoJobIDs - } - finishOtherTests := func(g Generator) { - for i := 2; i < 9; i++ { - job := nextJob(t, g, 1, invokerconn.TestJob, uint64(i)+1) - sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: job.ID, - Verdict: verdict.OK, - }) - require.Nil(t, sub) - require.Nil(t, err) - } - job := nextJob(t, g, 1, invokerconn.TestJob, 10) - sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: job.ID, - Verdict: verdict.OK, - }) - require.NotNil(t, sub) - require.Nil(t, err) - - require.Equal(t, verdict.OK, sub.Verdict) - require.Equal(t, 1., sub.Score) - for i, result := range sub.TestResults { - require.Equal(t, verdict.OK, result.Verdict) - require.Equal(t, uint64(i)+1, result.TestNumber) - } - } - - t.Run("right order", func(t *testing.T) { - g, firstTwoJobIDs := prepare() - for _, id := range firstTwoJobIDs { - sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: id, - Verdict: verdict.OK, - }) - require.Nil(t, sub) - require.Nil(t, err) - } - finishOtherTests(g) - }) - - t.Run("wrong order + both ok", func(t *testing.T) { - g, firstTwoJobIDs := prepare() - sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: firstTwoJobIDs[1], - Verdict: verdict.OK, - }) - require.Nil(t, sub) - require.Nil(t, err) - - sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: firstTwoJobIDs[0], - Verdict: verdict.OK, - }) - require.Nil(t, sub) - require.Nil(t, err) - - finishOtherTests(g) - }) - - t.Run("wrong order + 2nd fail", func(t *testing.T) { - g, firstTwoJobIDs := prepare() - sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: firstTwoJobIDs[1], - Verdict: verdict.WA, - }) - require.Nil(t, sub) - require.Nil(t, err) - - sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: firstTwoJobIDs[0], - Verdict: verdict.OK, - }) - require.NotNil(t, sub) - require.Nil(t, err) - - require.Equal(t, verdict.WA, sub.Verdict) - require.Equal(t, 0., sub.Score) - require.Equal(t, verdict.OK, sub.TestResults[0].Verdict) - require.Equal(t, verdict.WA, sub.TestResults[1].Verdict) - for i, result := range sub.TestResults[2:] { - require.Equal(t, verdict.SK, result.Verdict) - require.Equal(t, uint64(i)+3, result.TestNumber) - } - }) - - t.Run("wrong order + 1st fail", func(t *testing.T) { - g, firstTwoJobIDs := prepare() - sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: firstTwoJobIDs[1], - Verdict: verdict.OK, - }) - require.Nil(t, sub) - require.Nil(t, err) - - sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: firstTwoJobIDs[0], - Verdict: verdict.WA, - }) - require.NotNil(t, sub) - require.Nil(t, err) - - require.Equal(t, verdict.WA, sub.Verdict) - require.Equal(t, 0., sub.Score) - require.Equal(t, verdict.WA, sub.TestResults[0].Verdict) - require.Equal(t, verdict.SK, sub.TestResults[1].Verdict) - for i, result := range sub.TestResults[2:] { - require.Equal(t, verdict.SK, result.Verdict) - require.Equal(t, uint64(i)+3, result.TestNumber) - } - }) -} - -func TestFailedCompilation(t *testing.T) { - problem, submission := fixtureProblem(), fixtureSubmission() - g, err := NewGenerator(problem, submission) - require.Nil(t, err) - job := nextJob(t, g, 1, invokerconn.CompileJob, 0) - sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: job.ID, - Verdict: verdict.CE, - }) - require.NotNil(t, sub) - require.Nil(t, err) - - require.Equal(t, verdict.CE, sub.Verdict) - require.Equal(t, 0., sub.Score) - for i, result := range sub.TestResults { - require.Equal(t, verdict.SK, result.Verdict) - require.Equal(t, uint64(i)+1, result.TestNumber) - } -} - -func TestFinishSameJobTwice(t *testing.T) { - problem, submission := fixtureProblem(), fixtureSubmission() - g, err := NewGenerator(problem, submission) - require.Nil(t, err) - job := nextJob(t, g, 1, invokerconn.CompileJob, 0) - sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: job.ID, - Verdict: verdict.CE, - }) - require.NotNil(t, sub) - require.Nil(t, err) - - require.Equal(t, verdict.CE, sub.Verdict) - require.Equal(t, 0., sub.Score) - for i, result := range sub.TestResults { - require.Equal(t, verdict.SK, result.Verdict) - require.Equal(t, uint64(i)+1, result.TestNumber) - } - - sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ - JobID: job.ID, - Verdict: verdict.CE, - }) - require.Nil(t, sub) - require.NotNil(t, err) -} diff --git a/master/queue/jobgenerators/ioi_generator.go b/master/queue/jobgenerators/ioi_generator.go index bfd3c00..17105b4 100644 --- a/master/queue/jobgenerators/ioi_generator.go +++ b/master/queue/jobgenerators/ioi_generator.go @@ -3,6 +3,7 @@ package jobgenerators import ( "fmt" "github.com/google/uuid" + "github.com/xorcare/pointer" "slices" "sync" "testing_system/common/connectors/invokerconn" @@ -17,80 +18,201 @@ type IOIGenerator struct { mutex sync.Mutex submission *models.Submission problem *models.Problem - state state + state generatorState givenJobs map[string]*invokerconn.Job - groupNameToGroup map[string]*models.TestGroup - groupNamesToBeGiven []string - groupNameToDependencies map[string][]*models.TestGroup + // groupNameToOrigGroup should be set once at the creating of the generator and should not be changed further + groupNameToOrigGroup map[string]*models.TestGroup + // origGroupNamesToBeGiven should be set once at the creating of the generator and should not be changed further + origGroupNamesToBeGiven []string + + groupNameToGroup map[string]*models.TestGroup + groupNameToGroupResult map[string]*models.GroupResult + groupNamesToBeGiven []string } -func (i *IOIGenerator) finalizeResults() { +type problemWalkthroughResults struct { + order []string + hasCycle bool + used map[string]int +} + +func (i *IOIGenerator) walkThroughGroups(curGroup *models.TestGroup, result *problemWalkthroughResults) { + result.used[curGroup.Name] = 1 + for _, requiredGroupName := range curGroup.RequiredGroupNames { + requiredGroup := i.groupNameToGroup[requiredGroupName] + if val, ok := result.used[requiredGroup.Name]; !ok { + i.walkThroughGroups(requiredGroup, result) + } else if val == 1 { + result.hasCycle = true + } + } + result.order = append(result.order, curGroup.Name) + result.used[curGroup.Name] = 2 +} + +func (i *IOIGenerator) getWalkthroughResults() *problemWalkthroughResults { + result := &problemWalkthroughResults{ + order: make([]string, 0), + hasCycle: false, + used: make(map[string]int), + } for _, group := range i.groupNameToGroup { + if _, ok := result.used[group.Name]; !ok { + i.walkThroughGroups(group, result) + } + } + return result +} + +func (i *IOIGenerator) prepareGenerator() error { + problem := i.problem + if problem.ProblemType != models.ProblemTypeIOI { + return fmt.Errorf("problem %v is not an IOI problem", problem.ID) + } + // must copy groups in order to not break the problem + for _, group := range problem.TestGroups { + if _, ok := i.groupNameToGroup[group.Name]; ok { + return fmt.Errorf("group %v presented twice in problem", problem.ID) + } + group1 := group + i.groupNameToGroup[group1.Name] = &group1 + group2 := group1 + i.groupNameToOrigGroup[group2.Name] = &group2 + } + // each group with TestGroupScoringTypeEachTest must have TestScore + for _, group := range problem.TestGroups { + switch group.ScoringType { + case models.TestGroupScoringTypeComplete: + if group.Score == nil { + return fmt.Errorf("group %v in problem %v doesn't have Score", group.Name, problem.ID) + } + case models.TestGroupScoringTypeEachTest: + if group.TestScore == nil { + return fmt.Errorf("group %v in problem %v doesn't have TestScore", group.Name, problem.ID) + } + case models.TestGroupScoringTypeMin: + default: + return fmt.Errorf("unknown TestGroupScoringType %v", group.ScoringType) + } + } + { + // simple check that each test is in exactly one group + for testNumber := range i.problem.TestsNumber { + cnt := 0 + for _, group := range i.problem.TestGroups { + if uint64(group.FirstTest) <= testNumber+1 && testNumber+1 <= uint64(group.LastTest) { + cnt += 1 + } + } + if cnt != 1 { + return fmt.Errorf("test %v is presented in %v groups in problem %v", testNumber, problem.ID, cnt) + } + } + } + { + // check for cycles and build topsort of the groups + result := i.getWalkthroughResults() + if result.hasCycle { + return fmt.Errorf("cycle detected in problem %v", problem.ID) + } + i.groupNamesToBeGiven = result.order + i.origGroupNamesToBeGiven = result.order + } + { + testResults := make([]models.TestResult, 0, problem.TestsNumber) + for testNumber := range problem.TestsNumber { + testResults = append(testResults, models.TestResult{ + TestNumber: testNumber + 1, + Verdict: verdict.SK, + Time: 0, + Memory: 0, + }) + } + i.submission.TestResults = testResults + } + return nil +} + +func (i *IOIGenerator) finalizeResults() { + totalScore := 0.0 + i.submission.Verdict = verdict.OK + for _, groupName := range i.origGroupNamesToBeGiven { + group := i.groupNameToOrigGroup[groupName] + if _, ok := i.groupNameToGroupResult[group.Name]; !ok { + i.groupNameToGroupResult[group.Name] = &models.GroupResult{ + GroupName: group.Name, + Passed: false, + } + } + groupResult := i.groupNameToGroupResult[group.Name] + + allRequiredPassed := true + for _, requiredGroupName := range group.RequiredGroupNames { + if !i.groupNameToGroupResult[requiredGroupName].Passed { + allRequiredPassed = false + break + } + } + if !allRequiredPassed { + groupResult.Passed = false + groupResult.Points = 0. + for j := group.FirstTest - 1; j < group.LastTest; j++ { + i.submission.TestResults[j].Verdict = verdict.SK + i.submission.TestResults[j].Points = pointer.Float64(0.0) + i.submission.TestResults[j].Error = "" + } + i.groupNameToGroupResult[groupName] = groupResult + continue + } + + { + groupTestResults := i.submission.TestResults[group.FirstTest-1 : group.LastTest] + allOK := slices.IndexFunc(groupTestResults, func(result models.TestResult) bool { + return result.Verdict != verdict.OK + }) == -1 + groupResult.Passed = allOK + if !allOK { + i.submission.Verdict = verdict.PT + } + } + if groupResult.Passed && group.ScoringType == models.TestGroupScoringTypeComplete { + totalScore += *group.Score + groupResult.Points = *group.Score + } + switch group.ScoringType { case models.TestGroupScoringTypeComplete: setSkipped := false - for j := range i.submission.TestResults[group.FirstTest-1 : group.LastTest] { + for j := group.FirstTest - 1; j < group.LastTest; j++ { if setSkipped { i.submission.TestResults[j].Verdict = verdict.SK continue } - if i.submission.TestResults[j].Verdict == verdict.OK || i.submission.TestResults[j].Verdict == verdict.SK { + if i.submission.TestResults[j].Verdict == verdict.OK || + i.submission.TestResults[j].Verdict == verdict.SK { continue } setSkipped = true - i.submission.Verdict = i.submission.TestResults[j].Verdict - } - if !setSkipped { - if group.Score == nil { - logger.Panic("Group '%v' in problemId=%v has TypeComplete, but score is nil", - group.Name, i.problem.ID) - } - i.submission.TestResults[group.LastTest-1].Points = group.Score } case models.TestGroupScoringTypeEachTest: - // TODO + fallthrough case models.TestGroupScoringTypeMin: - // TODO - } - } - notOkInd := slices.IndexFunc(i.submission.TestResults, func(result models.TestResult) bool { - return result.Verdict != verdict.OK - }) - if notOkInd == -1 { - i.submission.Verdict = verdict.OK - } else { - i.submission.Verdict = i.submission.TestResults[notOkInd].Verdict - } - totalScore := 0. - for _, result := range i.submission.TestResults { - if result.Points != nil { - totalScore += *result.Points + for j := group.FirstTest - 1; j < group.LastTest; j++ { + groupResult.Points += *i.submission.TestResults[j].Points + totalScore += *i.submission.TestResults[j].Points + } } + i.groupNameToGroupResult[groupName] = groupResult } i.submission.Score = totalScore -} - -func (i *IOIGenerator) fetchGroupDependencies(curGroup *models.TestGroup) map[string]struct{} { - used := make(map[string]struct{}) - q := make([]*models.TestGroup, 0) - q = append(q, curGroup) - used[curGroup.Name] = struct{}{} - for len(q) > 0 { - group := q[0] - q = q[1:] - for _, nextGroup := range i.groupNameToDependencies[group.Name] { - if _, ok := used[nextGroup.Name]; !ok { - q = append(q, nextGroup) - used[nextGroup.Name] = struct{}{} - } - } + for _, group := range i.problem.TestGroups { + i.submission.GroupResults = append(i.submission.GroupResults, *i.groupNameToGroupResult[group.Name]) } - return used } func (i *IOIGenerator) testNumberToGroup(testNumber uint64) *models.TestGroup { - for _, group := range i.groupNameToGroup { + for _, group := range i.groupNameToOrigGroup { if group.FirstTest <= int(testNumber) && int(testNumber) <= group.LastTest { return group } @@ -122,19 +244,34 @@ func (i *IOIGenerator) NextJob() *invokerconn.Job { if i.state == compilationNotStarted { job.Type = invokerconn.CompileJob i.state = compilationStarted + i.givenJobs[job.ID] = job return job } - curGroupName := i.groupNamesToBeGiven[0] - group := i.groupNameToGroup[curGroupName] job.Type = invokerconn.TestJob - job.Test = uint64(group.FirstTest) - if group.FirstTest == group.LastTest { - i.groupNamesToBeGiven = i.groupNamesToBeGiven[1:] - } else { - group.FirstTest++ - } - i.givenJobs[job.ID] = job - return job + for len(i.groupNamesToBeGiven) > 0 { + curGroup := i.groupNamesToBeGiven[0] + shouldSkip := false + group := i.groupNameToGroup[curGroup] + for _, requiredGroupName := range group.RequiredGroupNames { + if result, ok := i.groupNameToGroupResult[requiredGroupName]; ok && !result.Passed { + shouldSkip = true + break + } + } + if shouldSkip { + i.groupNamesToBeGiven = i.groupNamesToBeGiven[1:] + continue + } + job.Test = uint64(group.FirstTest) + if group.FirstTest == group.LastTest { + i.groupNamesToBeGiven = i.groupNamesToBeGiven[1:] + } else { + group.FirstTest++ + } + i.givenJobs[job.ID] = job + return job + } + return nil } // compileJobCompleted must be done with acquired mutex @@ -147,6 +284,7 @@ func (i *IOIGenerator) compileJobCompleted(job *invokerconn.Job, result *masterc i.state = compilationFinished return nil, nil case verdict.CE: + i.finalizeResults() i.submission.Verdict = result.Verdict return i.submission, nil default: @@ -157,7 +295,7 @@ func (i *IOIGenerator) compileJobCompleted(job *invokerconn.Job, result *masterc // testJobCompleted must be done with acquired mutex func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn.InvokerJobResult) (*models.Submission, error) { if job.Type != invokerconn.TestJob { - return nil, fmt.Errorf("job type %s is not test job", job.ID) + return nil, fmt.Errorf("job type %v is not test job", job.ID) } group := i.testNumberToGroup(job.Test) if group == nil { @@ -165,47 +303,87 @@ func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn return nil, nil // just for goland to chill } + if group.ScoringType == models.TestGroupScoringTypeEachTest && + (result.Points == nil || *result.Points > *group.TestScore) { + if len(result.Error) > 0 { + result.Error += "; " + } + result.Error += fmt.Sprintf("checker returned score=%v, but testScore=%v in problemId=%v, group=%v", + result.Points, *group.TestScore, job.ID, i.problem.ID) + result.Verdict = verdict.CF + } + if group.ScoringType == models.TestGroupScoringTypeMin && result.Points == nil { + if len(result.Error) > 0 { + result.Error += "; " + } + result.Error += fmt.Sprintf("checker returned score=%v, but testScore=%v in problemId=%v, group=%v", + result.Points, *group.TestScore, job.ID, i.problem.ID) + result.Verdict = verdict.CF + } + i.submission.TestResults[job.Test-1].Verdict = result.Verdict switch result.Verdict { case verdict.OK: - if group.ScoringType == models.TestGroupScoringTypeEachTest { - if group.TestScore == nil { - logger.Panic("Group '%v' has type EachTest, but TestScore is nil in problemId=%v", - group.Name, i.problem.ID) - } - i.submission.TestResults[job.Test-1].Points = group.TestScore - } else if group.ScoringType == models.TestGroupScoringTypeMin { - if result.Points == nil { - logger.Panic("Group '%v' has type Min, but checker didn't set points in problemId=%v, jobId=%v", - group.Name, i.problem.ID, job.ID) + case verdict.CF: + i.submission.TestResults[job.Test-1].Error = result.Error + fallthrough + case verdict.PT, verdict.WA, verdict.RT, verdict.ML, verdict.TL, verdict.WL, verdict.SE: + if _, ok := i.groupNameToGroupResult[group.Name]; !ok { + i.groupNameToGroupResult[group.Name] = &models.GroupResult{ + GroupName: group.Name, + Passed: false, } - i.submission.TestResults[job.Test-1].Points = result.Points } - if len(i.givenJobs) == 0 && len(i.groupNamesToBeGiven) == 0 { - i.finalizeResults() - return i.submission, nil + default: + return nil, fmt.Errorf("unknown verdict for test %v in job %v", job.Test, job.ID) + } + + if result.Verdict == verdict.OK || result.Verdict == verdict.PT { + switch group.ScoringType { + case models.TestGroupScoringTypeEachTest, models.TestGroupScoringTypeMin: + i.submission.TestResults[job.Test-1].Points = group.Score + case models.TestGroupScoringTypeComplete: } - return nil, nil - case verdict.PT, verdict.WA, verdict.RT, verdict.ML, verdict.TL, verdict.WL, verdict.SE, verdict.CF: - dependencies := i.fetchGroupDependencies(group) + } + + /* + scenario: + group1: complete, tests: 1-1 + group2: complete, tests: 2-2 + 1. NextJob returns the test from the first group + 2. WA it + 3. It is expected that we already know testing result + so we have to go through all the groups and check for their required groups if they are not passed already + */ + { newGroupNamesToBeGiven := make([]string, 0) - for _, s := range i.groupNamesToBeGiven { - if _, ok := dependencies[s]; !ok { - newGroupNamesToBeGiven = append(newGroupNamesToBeGiven, s) - } else if s == group.Name && group.ScoringType != models.TestGroupScoringTypeComplete { - newGroupNamesToBeGiven = append(newGroupNamesToBeGiven, s) + for _, groupName := range i.groupNamesToBeGiven { + group = i.groupNameToOrigGroup[groupName] + shouldTest := true + for _, requiredGroupName := range group.RequiredGroupNames { + if _, ok := i.groupNameToGroupResult[requiredGroupName]; ok { + // if the group is presented in this map, then it is failed already + shouldTest = false + break + } + } + if _, ok := i.groupNameToGroupResult[groupName]; ok && + group.ScoringType == models.TestGroupScoringTypeComplete { + shouldTest = false + } + if shouldTest { + newGroupNamesToBeGiven = append(newGroupNamesToBeGiven, groupName) } } i.groupNamesToBeGiven = newGroupNamesToBeGiven - if len(i.givenJobs) == 0 && len(i.groupNamesToBeGiven) == 0 { - i.finalizeResults() - return i.submission, nil - } - return nil, nil - default: - return nil, fmt.Errorf("unknown verdict for testing completed: %v", result.Verdict) } + + if len(i.givenJobs) == 0 && len(i.groupNamesToBeGiven) == 0 { + i.finalizeResults() + return i.submission, nil + } + return nil, nil } func (i *IOIGenerator) JobCompleted(jobResult *masterconn.InvokerJobResult) (*models.Submission, error) { @@ -227,5 +405,24 @@ func (i *IOIGenerator) JobCompleted(jobResult *masterconn.InvokerJobResult) (*mo } func NewIOIGenerator(problem *models.Problem, submission *models.Submission) (Generator, error) { - return &IOIGenerator{}, nil + id, err := uuid.NewV7() + if err != nil { + logger.Panic("Can't generate generator id: %w", err) + } + generator := &IOIGenerator{ + id: id.String(), + submission: submission, + problem: problem, + state: compilationNotStarted, + givenJobs: make(map[string]*invokerconn.Job), + groupNameToGroupResult: make(map[string]*models.GroupResult), + // these fields will be filled in prepareGenerator + groupNameToOrigGroup: make(map[string]*models.TestGroup), + groupNameToGroup: make(map[string]*models.TestGroup), + groupNamesToBeGiven: nil, + } + if err = generator.prepareGenerator(); err != nil { + return nil, err + } + return generator, nil } diff --git a/master/queue/jobgenerators/ioi_generator_test.go b/master/queue/jobgenerators/ioi_generator_test.go deleted file mode 100644 index 1601935..0000000 --- a/master/queue/jobgenerators/ioi_generator_test.go +++ /dev/null @@ -1 +0,0 @@ -package jobgenerators From 9d12626e5cee66bf212cae4361a012ceae0e3f2b Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Sat, 19 Apr 2025 02:24:16 +0300 Subject: [PATCH 03/10] i hate IOI problems --- common/db/models/problem.go | 14 +- master/queue/jobgenerators/generator_test.go | 257 ++++++++++++- master/queue/jobgenerators/ioi_generator.go | 366 +++++++++++-------- 3 files changed, 462 insertions(+), 175 deletions(-) diff --git a/common/db/models/problem.go b/common/db/models/problem.go index 954f842..066935d 100644 --- a/common/db/models/problem.go +++ b/common/db/models/problem.go @@ -44,15 +44,15 @@ const ( type TestGroup struct { Name string `json:"name" yaml:"name"` - FirstTest int `json:"first_test" yaml:"first_test"` - LastTest int `json:"last_test" yaml:"last_test"` + FirstTest int `json:"FirstTest" yaml:"FirstTest"` + LastTest int `json:"LastTest" yaml:"LastTest"` // TestScore meaningful only in case of TestGroupScoringTypeEachTest - TestScore *float64 `json:"test_score" yaml:"test_score"` + TestScore *float64 `json:"TestScore" yaml:"TestScore"` // Score meaningful only in case of TestGroupScoringTypeComplete - Score *float64 `json:"score" yaml:"score"` - ScoringType TestGroupScoringType `json:"scoring_type" yaml:"scoring_type"` - FeedbackType TestGroupFeedbackType `json:"feedback_type" yaml:"feedback_type"` - RequiredGroupNames []string `json:"required_groups" yaml:"required_groups"` + Score *float64 `json:"Score" yaml:"Score"` + ScoringType TestGroupScoringType `json:"ScoringType" yaml:"ScoringType"` + FeedbackType TestGroupFeedbackType `json:"FeedbackType" yaml:"FeedbackType"` + RequiredGroupNames []string `json:"RequiredGroupNames" yaml:"RequiredGroupNames"` } func (t TestGroup) Value() (driver.Value, error) { diff --git a/master/queue/jobgenerators/generator_test.go b/master/queue/jobgenerators/generator_test.go index 1361e86..55c1074 100644 --- a/master/queue/jobgenerators/generator_test.go +++ b/master/queue/jobgenerators/generator_test.go @@ -26,16 +26,16 @@ func fixtureSubmission() *models.Submission { func nextJob(t *testing.T, g Generator, SubmitID uint, jobType invokerconn.JobType, test uint64) *invokerconn.Job { job := g.NextJob() - assert.NotNil(t, job) - assert.Equal(t, job.Type, jobType) - assert.Equal(t, SubmitID, job.SubmitID) - assert.Equal(t, test, job.Test) + require.NotNil(t, job) + require.Equal(t, job.Type, jobType) + require.Equal(t, SubmitID, job.SubmitID) + require.Equal(t, test, job.Test) return job } func noJobs(t *testing.T, g Generator) { job := g.NextJob() - assert.Nil(t, job) + require.Nil(t, job) } func TestICPCGenerator(t *testing.T) { @@ -413,13 +413,13 @@ func TestIOIGenerator(t *testing.T) { require.Equal(t, uint64(i)+1, result.TestNumber) } require.Equal(t, wasProblem, problemWithOneGroup) - require.Equal(t, sub.GroupResults, models.GroupResults{ + require.Equal(t, models.GroupResults{ { GroupName: "group1", Points: 0., Passed: false, }, - }) + }, sub.GroupResults) }) t.Run("Straight task finishing", func(t *testing.T) { @@ -711,4 +711,247 @@ func TestIOIGenerator(t *testing.T) { }) }) }) + + t.Run("Fails in TestGroupScoringTypeEachTest", func(t *testing.T) { + t.Run("WA in the middle of the group", func(t *testing.T) { + problem := models.Problem{ + ProblemType: models.ProblemTypeIOI, + TestsNumber: 3, + TestGroups: []models.TestGroup{ + { + Name: "group1", + FirstTest: 1, + LastTest: 3, + TestScore: pointer.Float64(20), + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + }, + } + g, err := NewIOIGenerator(&problem, fixtureSubmission()) + require.NoError(t, err) + job := nextJob(t, g, 1, invokerconn.CompileJob, 0) + require.NotNil(t, job) + require.Nil(t, g.NextJob()) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.NoError(t, err) + require.Nil(t, sub) + // test + job = nextJob(t, g, 1, invokerconn.TestJob, 1) + require.NotNil(t, job) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + job2 := nextJob(t, g, 1, invokerconn.TestJob, 2) + require.NotNil(t, job2) + job3 := nextJob(t, g, 1, invokerconn.TestJob, 3) + require.NotNil(t, job3) + require.Nil(t, g.NextJob()) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job2.ID, + Verdict: verdict.WA, + }) + require.NoError(t, err) + require.Nil(t, sub) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job3.ID, + Verdict: verdict.OK, + }) + require.NoError(t, err) + require.NotNil(t, sub) + + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 40., sub.Score) + require.Equal(t, verdict.OK, sub.TestResults[0].Verdict) + require.Equal(t, verdict.WA, sub.TestResults[1].Verdict) + require.Equal(t, verdict.OK, sub.TestResults[2].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 40., + Passed: false, + }, + }) + }) + }) + + t.Run("TestGroupScoringTypeMin", func(t *testing.T) { + prepare := func() models.Problem { + return models.Problem{ + ProblemType: models.ProblemTypeIOI, + TestsNumber: 3, + TestGroups: []models.TestGroup{ + { + Name: "group1", + FirstTest: 1, + LastTest: 3, + Score: pointer.Float64(20), + ScoringType: models.TestGroupScoringTypeMin, + RequiredGroupNames: make([]string, 0), + }, + }, + } + } + + t.Run("WA in the middle of the group", func(t *testing.T) { + problem := prepare() + g, err := NewIOIGenerator(&problem, fixtureSubmission()) + require.NoError(t, err) + job := nextJob(t, g, 1, invokerconn.CompileJob, 0) + require.NotNil(t, job) + require.Nil(t, g.NextJob()) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.NoError(t, err) + require.Nil(t, sub) + // test + job = nextJob(t, g, 1, invokerconn.TestJob, 1) + require.NotNil(t, job) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + job2 := nextJob(t, g, 1, invokerconn.TestJob, 2) + require.NotNil(t, job2) + job3 := nextJob(t, g, 1, invokerconn.TestJob, 3) + require.NotNil(t, job3) + require.Nil(t, g.NextJob()) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job2.ID, + Verdict: verdict.WA, + }) + require.NoError(t, err) + require.Nil(t, sub) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job3.ID, + Verdict: verdict.OK, + }) + require.NoError(t, err) + require.NotNil(t, sub) + + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 0., sub.Score) + require.Equal(t, verdict.OK, sub.TestResults[0].Verdict) + require.Equal(t, verdict.WA, sub.TestResults[1].Verdict) + require.Equal(t, verdict.SK, sub.TestResults[2].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 0., + Passed: false, + }, + }) + }) + + t.Run("PT in the middle of the group", func(t *testing.T) { + problem := prepare() + g, err := NewIOIGenerator(&problem, fixtureSubmission()) + require.NoError(t, err) + job := nextJob(t, g, 1, invokerconn.CompileJob, 0) + require.NotNil(t, job) + require.Nil(t, g.NextJob()) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.NoError(t, err) + require.Nil(t, sub) + // test + job = nextJob(t, g, 1, invokerconn.TestJob, 1) + require.NotNil(t, job) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + job2 := nextJob(t, g, 1, invokerconn.TestJob, 2) + require.NotNil(t, job2) + job3 := nextJob(t, g, 1, invokerconn.TestJob, 3) + require.NotNil(t, job3) + require.Nil(t, g.NextJob()) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job2.ID, + Verdict: verdict.PT, + Points: pointer.Float64(10), + }) + require.NoError(t, err) + require.Nil(t, sub) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job3.ID, + Verdict: verdict.OK, + }) + require.NoError(t, err) + require.NotNil(t, sub) + + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 10., sub.Score) + require.Equal(t, verdict.OK, sub.TestResults[0].Verdict) + require.Equal(t, verdict.PT, sub.TestResults[1].Verdict) + require.Equal(t, verdict.OK, sub.TestResults[2].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 10., + Passed: false, + }, + }) + }) + + t.Run("no fails", func(t *testing.T) { + problem := prepare() + g, err := NewIOIGenerator(&problem, fixtureSubmission()) + require.NoError(t, err) + job := nextJob(t, g, 1, invokerconn.CompileJob, 0) + require.NotNil(t, job) + require.Nil(t, g.NextJob()) + sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.NoError(t, err) + require.Nil(t, sub) + // test + job = nextJob(t, g, 1, invokerconn.TestJob, 1) + require.NotNil(t, job) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.OK, + }) + job2 := nextJob(t, g, 1, invokerconn.TestJob, 2) + require.NotNil(t, job2) + job3 := nextJob(t, g, 1, invokerconn.TestJob, 3) + require.NotNil(t, job3) + require.Nil(t, g.NextJob()) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job2.ID, + Verdict: verdict.OK, + }) + require.NoError(t, err) + require.Nil(t, sub) + sub, err = g.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job3.ID, + Verdict: verdict.OK, + }) + require.NoError(t, err) + require.NotNil(t, sub) + + require.Equal(t, verdict.OK, sub.Verdict) + require.Equal(t, 20., sub.Score) + require.Equal(t, verdict.OK, sub.TestResults[0].Verdict) + require.Equal(t, verdict.OK, sub.TestResults[1].Verdict) + require.Equal(t, verdict.OK, sub.TestResults[2].Verdict) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 20., + Passed: true, + }, + }) + }) + }) } diff --git a/master/queue/jobgenerators/ioi_generator.go b/master/queue/jobgenerators/ioi_generator.go index 17105b4..2025c93 100644 --- a/master/queue/jobgenerators/ioi_generator.go +++ b/master/queue/jobgenerators/ioi_generator.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/google/uuid" "github.com/xorcare/pointer" + "math" "slices" "sync" "testing_system/common/connectors/invokerconn" @@ -26,40 +27,59 @@ type IOIGenerator struct { // origGroupNamesToBeGiven should be set once at the creating of the generator and should not be changed further origGroupNamesToBeGiven []string - groupNameToGroup map[string]*models.TestGroup - groupNameToGroupResult map[string]*models.GroupResult - groupNamesToBeGiven []string + groupNameToGroupInfo map[string]*internalGroupInfo + groupNamesToBeGiven []string + testNumberToGroupName map[uint64]string +} + +type groupState int + +const ( + onlyOK groupState = iota + okOrPT + hasFails +) + +type internalGroupInfo struct { + group models.TestGroup + minPoints float64 + sumPoints float64 + testsPasses int + groupState groupState } type problemWalkthroughResults struct { - order []string - hasCycle bool - used map[string]int + order []string + used map[string]int + err error } -func (i *IOIGenerator) walkThroughGroups(curGroup *models.TestGroup, result *problemWalkthroughResults) { - result.used[curGroup.Name] = 1 - for _, requiredGroupName := range curGroup.RequiredGroupNames { - requiredGroup := i.groupNameToGroup[requiredGroupName] - if val, ok := result.used[requiredGroup.Name]; !ok { - i.walkThroughGroups(requiredGroup, result) +func (i *IOIGenerator) walkThroughGroups(groupInfo *internalGroupInfo, result *problemWalkthroughResults) { + result.used[groupInfo.group.Name] = 1 + for _, requiredGroupName := range groupInfo.group.RequiredGroupNames { + requiredGroupInfo := i.groupNameToGroupInfo[requiredGroupName] + if requiredGroupInfo.group.ScoringType == models.TestGroupScoringTypeEachTest || + requiredGroupInfo.group.ScoringType == models.TestGroupScoringTypeMin { + result.err = fmt.Errorf("group can't depend on group with scoring type each test or min") + } else if val, ok := result.used[requiredGroupInfo.group.Name]; !ok { + i.walkThroughGroups(requiredGroupInfo, result) } else if val == 1 { - result.hasCycle = true + result.err = fmt.Errorf("cycle detected in dependencies") } } - result.order = append(result.order, curGroup.Name) - result.used[curGroup.Name] = 2 + result.order = append(result.order, groupInfo.group.Name) + result.used[groupInfo.group.Name] = 2 } func (i *IOIGenerator) getWalkthroughResults() *problemWalkthroughResults { result := &problemWalkthroughResults{ - order: make([]string, 0), - hasCycle: false, - used: make(map[string]int), + order: make([]string, 0), + used: make(map[string]int), + err: nil, } - for _, group := range i.groupNameToGroup { - if _, ok := result.used[group.Name]; !ok { - i.walkThroughGroups(group, result) + for _, groupInfo := range i.groupNameToGroupInfo { + if _, ok := result.used[groupInfo.group.Name]; !ok { + i.walkThroughGroups(groupInfo, result) } } return result @@ -72,18 +92,26 @@ func (i *IOIGenerator) prepareGenerator() error { } // must copy groups in order to not break the problem for _, group := range problem.TestGroups { - if _, ok := i.groupNameToGroup[group.Name]; ok { + if _, ok := i.groupNameToGroupInfo[group.Name]; ok { return fmt.Errorf("group %v presented twice in problem", problem.ID) } + i.groupNameToGroupInfo[group.Name] = &internalGroupInfo{ + group: group, + minPoints: math.Inf(1), + sumPoints: 0, + testsPasses: 0, + groupState: onlyOK, + } group1 := group - i.groupNameToGroup[group1.Name] = &group1 - group2 := group1 - i.groupNameToOrigGroup[group2.Name] = &group2 + i.groupNameToOrigGroup[group1.Name] = &group1 + for testNumber := group.FirstTest - 1; testNumber < group.LastTest; testNumber++ { + i.testNumberToGroupName[uint64(testNumber)] = group.Name + } } // each group with TestGroupScoringTypeEachTest must have TestScore for _, group := range problem.TestGroups { switch group.ScoringType { - case models.TestGroupScoringTypeComplete: + case models.TestGroupScoringTypeComplete, models.TestGroupScoringTypeMin: if group.Score == nil { return fmt.Errorf("group %v in problem %v doesn't have Score", group.Name, problem.ID) } @@ -91,7 +119,6 @@ func (i *IOIGenerator) prepareGenerator() error { if group.TestScore == nil { return fmt.Errorf("group %v in problem %v doesn't have TestScore", group.Name, problem.ID) } - case models.TestGroupScoringTypeMin: default: return fmt.Errorf("unknown TestGroupScoringType %v", group.ScoringType) } @@ -111,10 +138,10 @@ func (i *IOIGenerator) prepareGenerator() error { } } { - // check for cycles and build topsort of the groups + // check for cycles and build order of groups result := i.getWalkthroughResults() - if result.hasCycle { - return fmt.Errorf("cycle detected in problem %v", problem.ID) + if result.err != nil { + return result.err } i.groupNamesToBeGiven = result.order i.origGroupNamesToBeGiven = result.order @@ -134,90 +161,94 @@ func (i *IOIGenerator) prepareGenerator() error { return nil } -func (i *IOIGenerator) finalizeResults() { +// finalizeResultsAfterTestJobCompleted must be done with acquired mutex and only ONCE (in case of no CE) +func (i *IOIGenerator) finalizeResultsAfterTestJobCompleted() { totalScore := 0.0 i.submission.Verdict = verdict.OK for _, groupName := range i.origGroupNamesToBeGiven { - group := i.groupNameToOrigGroup[groupName] - if _, ok := i.groupNameToGroupResult[group.Name]; !ok { - i.groupNameToGroupResult[group.Name] = &models.GroupResult{ - GroupName: group.Name, - Passed: false, + groupInfo := i.groupNameToGroupInfo[groupName] + origGroup := i.groupNameToOrigGroup[groupName] + // if at least one required group is failed, then current group is fully SK + haveFail := false + for _, requiredGroupName := range origGroup.RequiredGroupNames { + if i.groupNameToGroupInfo[requiredGroupName].groupState == hasFails { + haveFail = true } } - groupResult := i.groupNameToGroupResult[group.Name] - - allRequiredPassed := true - for _, requiredGroupName := range group.RequiredGroupNames { - if !i.groupNameToGroupResult[requiredGroupName].Passed { - allRequiredPassed = false - break + if haveFail { + for testNumber := origGroup.FirstTest - 1; testNumber < origGroup.LastTest; testNumber++ { + i.submission.TestResults[testNumber].Verdict = verdict.SK + i.submission.TestResults[testNumber].Points = pointer.Float64(0) } - } - if !allRequiredPassed { - groupResult.Passed = false - groupResult.Points = 0. - for j := group.FirstTest - 1; j < group.LastTest; j++ { - i.submission.TestResults[j].Verdict = verdict.SK - i.submission.TestResults[j].Points = pointer.Float64(0.0) - i.submission.TestResults[j].Error = "" - } - i.groupNameToGroupResult[groupName] = groupResult + i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ + GroupName: groupName, + Points: 0.0, + Passed: false, + }) continue } - - { - groupTestResults := i.submission.TestResults[group.FirstTest-1 : group.LastTest] - allOK := slices.IndexFunc(groupTestResults, func(result models.TestResult) bool { - return result.Verdict != verdict.OK - }) == -1 - groupResult.Passed = allOK - if !allOK { - i.submission.Verdict = verdict.PT - } + // otherwise calculate score for this group and set appropriate verdicts + if groupInfo.groupState != onlyOK { + i.submission.Verdict = verdict.PT } - if groupResult.Passed && group.ScoringType == models.TestGroupScoringTypeComplete { - totalScore += *group.Score - groupResult.Points = *group.Score - } - - switch group.ScoringType { - case models.TestGroupScoringTypeComplete: - setSkipped := false - for j := group.FirstTest - 1; j < group.LastTest; j++ { - if setSkipped { - i.submission.TestResults[j].Verdict = verdict.SK + curPoints := 0.0 + setSKVerdicts := func(notAcceptableVerdicts ...verdict.Verdict) { + // should set SK for all the tests after the first one without OK + setSK := false + for testNumber := origGroup.FirstTest - 1; testNumber < origGroup.LastTest; testNumber++ { + if !slices.Contains(notAcceptableVerdicts, i.submission.TestResults[testNumber].Verdict) { + setSK = true continue } - if i.submission.TestResults[j].Verdict == verdict.OK || - i.submission.TestResults[j].Verdict == verdict.SK { - continue + if setSK { + i.submission.TestResults[testNumber].Verdict = verdict.SK } - setSkipped = true + } + } + switch origGroup.ScoringType { + case models.TestGroupScoringTypeComplete: + if groupInfo.groupState == onlyOK { + curPoints = *origGroup.Score + } else { + setSKVerdicts(verdict.OK) } case models.TestGroupScoringTypeEachTest: - fallthrough + curPoints = groupInfo.sumPoints case models.TestGroupScoringTypeMin: - for j := group.FirstTest - 1; j < group.LastTest; j++ { - groupResult.Points += *i.submission.TestResults[j].Points - totalScore += *i.submission.TestResults[j].Points + if groupInfo.groupState == hasFails { + setSKVerdicts(verdict.OK, verdict.PT) + curPoints = 0.0 + } else { + if groupInfo.minPoints != math.Inf(1) { + curPoints = groupInfo.minPoints + } } } - i.groupNameToGroupResult[groupName] = groupResult + totalScore += curPoints + i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ + GroupName: groupName, + Points: curPoints, + Passed: groupInfo.groupState == onlyOK, + }) } i.submission.Score = totalScore - for _, group := range i.problem.TestGroups { - i.submission.GroupResults = append(i.submission.GroupResults, *i.groupNameToGroupResult[group.Name]) - } } -func (i *IOIGenerator) testNumberToGroup(testNumber uint64) *models.TestGroup { - for _, group := range i.groupNameToOrigGroup { - if group.FirstTest <= int(testNumber) && int(testNumber) <= group.LastTest { - return group +// finalizeResultsAfterCompileJobFailed must be done with acquired mutex and only ONCE (in case of CE) +func (i *IOIGenerator) finalizeResultsAfterCompileJobFailed() { + i.submission.Verdict = verdict.CE + for _, groupName := range i.origGroupNamesToBeGiven { + origGroup := i.groupNameToOrigGroup[groupName] + for testNumber := origGroup.FirstTest - 1; testNumber < origGroup.LastTest; testNumber++ { + i.submission.TestResults[testNumber].Verdict = verdict.SK } + i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ + GroupName: groupName, + Points: 0.0, + Passed: false, + }) } - return nil + i.submission.Score = 0.0 } func (i *IOIGenerator) ID() string { @@ -248,25 +279,13 @@ func (i *IOIGenerator) NextJob() *invokerconn.Job { return job } job.Type = invokerconn.TestJob - for len(i.groupNamesToBeGiven) > 0 { - curGroup := i.groupNamesToBeGiven[0] - shouldSkip := false - group := i.groupNameToGroup[curGroup] - for _, requiredGroupName := range group.RequiredGroupNames { - if result, ok := i.groupNameToGroupResult[requiredGroupName]; ok && !result.Passed { - shouldSkip = true - break - } - } - if shouldSkip { + for len(i.groupNameToGroupInfo) > 0 { + groupName := i.groupNamesToBeGiven[0] + groupInfo := i.groupNameToGroupInfo[groupName] + job.Test = uint64(groupInfo.group.FirstTest) + groupInfo.group.FirstTest++ + if groupInfo.group.FirstTest > groupInfo.group.LastTest { i.groupNamesToBeGiven = i.groupNamesToBeGiven[1:] - continue - } - job.Test = uint64(group.FirstTest) - if group.FirstTest == group.LastTest { - i.groupNamesToBeGiven = i.groupNamesToBeGiven[1:] - } else { - group.FirstTest++ } i.givenJobs[job.ID] = job return job @@ -284,8 +303,7 @@ func (i *IOIGenerator) compileJobCompleted(job *invokerconn.Job, result *masterc i.state = compilationFinished return nil, nil case verdict.CE: - i.finalizeResults() - i.submission.Verdict = result.Verdict + i.finalizeResultsAfterCompileJobFailed() return i.submission, nil default: return nil, fmt.Errorf("unknown verdict for compilation completed: %v", result.Verdict) @@ -297,60 +315,86 @@ func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn if job.Type != invokerconn.TestJob { return nil, fmt.Errorf("job type %v is not test job", job.ID) } - group := i.testNumberToGroup(job.Test) - if group == nil { - logger.Panic("Can't get group of test %v in job %v, problemId=%v", job.Test, job.ID, i.problem.ID) - return nil, nil // just for goland to chill - } + groupName := i.testNumberToGroupName[job.Test-1] + groupInfo := i.groupNameToGroupInfo[groupName] - if group.ScoringType == models.TestGroupScoringTypeEachTest && - (result.Points == nil || *result.Points > *group.TestScore) { + // a lot of checks for fucking IOI problems + setResultCF := func(checkerScore *float64, problemId uint, group string) { if len(result.Error) > 0 { result.Error += "; " } - result.Error += fmt.Sprintf("checker returned score=%v, but testScore=%v in problemId=%v, group=%v", - result.Points, *group.TestScore, job.ID, i.problem.ID) + result.Error += fmt.Sprintf( + "checker returned score=%v, but testScore=%v and score=%v in problemId=%v, group=%v, type=%v", + checkerScore, groupInfo.group.TestScore, groupInfo.group.Score, + problemId, group, groupInfo.group.ScoringType) result.Verdict = verdict.CF - } - if group.ScoringType == models.TestGroupScoringTypeMin && result.Points == nil { - if len(result.Error) > 0 { - result.Error += "; " + result.Points = pointer.Float64(0) + } + if groupInfo.group.ScoringType == models.TestGroupScoringTypeEachTest { + if result.Points == nil { + if result.Verdict == verdict.OK { + result.Points = groupInfo.group.TestScore + } else if result.Verdict == verdict.PT { + setResultCF(result.Points, i.problem.ID, groupName) + } + } else if *result.Points > *groupInfo.group.TestScore { + setResultCF(result.Points, i.problem.ID, groupName) + } + } else if groupInfo.group.ScoringType == models.TestGroupScoringTypeMin { + if result.Points == nil { + if result.Verdict == verdict.OK { + result.Points = groupInfo.group.Score + } else if result.Verdict == verdict.PT { + setResultCF(result.Points, i.problem.ID, groupName) + } + } else if *result.Points > *groupInfo.group.Score { + setResultCF(result.Points, i.problem.ID, groupName) + } + } else if groupInfo.group.ScoringType == models.TestGroupScoringTypeComplete { + if result.Verdict == verdict.PT { + result.Verdict = verdict.CF + if len(result.Error) > 0 { + result.Error += "; " + } + result.Error += fmt.Sprintf("checker returned PT on test=%v in problemId=%v, group=%v, type=%v", + job.Test, i.problem.ID, groupName, groupInfo.group.ScoringType) } - result.Error += fmt.Sprintf("checker returned score=%v, but testScore=%v in problemId=%v, group=%v", - result.Points, *group.TestScore, job.ID, i.problem.ID) - result.Verdict = verdict.CF } i.submission.TestResults[job.Test-1].Verdict = result.Verdict + if result.Points != nil { + groupInfo.minPoints = min(groupInfo.minPoints, *result.Points) + groupInfo.sumPoints += *result.Points + } + if result.Statistics != nil { + i.submission.TestResults[job.Test-1].Time = result.Statistics.Time + i.submission.TestResults[job.Test-1].Memory = result.Statistics.Memory + } switch result.Verdict { + case verdict.PT: + if groupInfo.groupState != hasFails { + groupInfo.groupState = okOrPT + } + fallthrough case verdict.OK: + if groupInfo.group.ScoringType == models.TestGroupScoringTypeEachTest || + groupInfo.group.ScoringType == models.TestGroupScoringTypeMin { + i.submission.TestResults[job.Test-1].Points = result.Points + } case verdict.CF: i.submission.TestResults[job.Test-1].Error = result.Error fallthrough - case verdict.PT, verdict.WA, verdict.RT, verdict.ML, verdict.TL, verdict.WL, verdict.SE: - if _, ok := i.groupNameToGroupResult[group.Name]; !ok { - i.groupNameToGroupResult[group.Name] = &models.GroupResult{ - GroupName: group.Name, - Passed: false, - } - } + case verdict.WA, verdict.RT, verdict.ML, verdict.TL, verdict.WL, verdict.SE: + groupInfo.groupState = hasFails default: return nil, fmt.Errorf("unknown verdict for test %v in job %v", job.Test, job.ID) } - if result.Verdict == verdict.OK || result.Verdict == verdict.PT { - switch group.ScoringType { - case models.TestGroupScoringTypeEachTest, models.TestGroupScoringTypeMin: - i.submission.TestResults[job.Test-1].Points = group.Score - case models.TestGroupScoringTypeComplete: - } - } - /* scenario: - group1: complete, tests: 1-1 - group2: complete, tests: 2-2 + group1: any scoring type, tests: 1-1 + group2: any scoring type, tests: 2-2 1. NextJob returns the test from the first group 2. WA it 3. It is expected that we already know testing result @@ -358,18 +402,17 @@ func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn */ { newGroupNamesToBeGiven := make([]string, 0) - for _, groupName := range i.groupNamesToBeGiven { - group = i.groupNameToOrigGroup[groupName] + for _, groupName = range i.groupNamesToBeGiven { + groupInfo = i.groupNameToGroupInfo[groupName] shouldTest := true - for _, requiredGroupName := range group.RequiredGroupNames { - if _, ok := i.groupNameToGroupResult[requiredGroupName]; ok { - // if the group is presented in this map, then it is failed already + for _, requiredGroupName := range groupInfo.group.RequiredGroupNames { + if i.groupNameToGroupInfo[requiredGroupName].groupState != onlyOK { shouldTest = false break } } - if _, ok := i.groupNameToGroupResult[groupName]; ok && - group.ScoringType == models.TestGroupScoringTypeComplete { + // this is the only case, since we assume that checker can not return PT in type complete + if groupInfo.groupState == hasFails { shouldTest = false } if shouldTest { @@ -380,7 +423,7 @@ func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn } if len(i.givenJobs) == 0 && len(i.groupNamesToBeGiven) == 0 { - i.finalizeResults() + i.finalizeResultsAfterTestJobCompleted() return i.submission, nil } return nil, nil @@ -410,16 +453,17 @@ func NewIOIGenerator(problem *models.Problem, submission *models.Submission) (Ge logger.Panic("Can't generate generator id: %w", err) } generator := &IOIGenerator{ - id: id.String(), - submission: submission, - problem: problem, - state: compilationNotStarted, - givenJobs: make(map[string]*invokerconn.Job), - groupNameToGroupResult: make(map[string]*models.GroupResult), + id: id.String(), + submission: submission, + problem: problem, + state: compilationNotStarted, + givenJobs: make(map[string]*invokerconn.Job), + groupNameToGroupInfo: make(map[string]*internalGroupInfo), // these fields will be filled in prepareGenerator - groupNameToOrigGroup: make(map[string]*models.TestGroup), - groupNameToGroup: make(map[string]*models.TestGroup), - groupNamesToBeGiven: nil, + groupNameToOrigGroup: make(map[string]*models.TestGroup), + groupNamesToBeGiven: nil, + origGroupNamesToBeGiven: nil, + testNumberToGroupName: make(map[uint64]string), } if err = generator.prepareGenerator(); err != nil { return nil, err From dd3a0a8e9d1a1872263e90e307697a920782e7e1 Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Thu, 1 May 2025 13:08:56 +0300 Subject: [PATCH 04/10] new generation of ioi generators --- common/db/models/problem.go | 10 +- master/queue/jobgenerators/generator_test.go | 61 +- master/queue/jobgenerators/ioi_generator.go | 591 +++++++++---------- 3 files changed, 320 insertions(+), 342 deletions(-) diff --git a/common/db/models/problem.go b/common/db/models/problem.go index 066935d..736f9c9 100644 --- a/common/db/models/problem.go +++ b/common/db/models/problem.go @@ -22,7 +22,7 @@ const ( ProblemTypeIOI ) const ( - // TestGroupScoringTypeComplete means that group costs TestGroup.Score (all the tests should be OK) + // TestGroupScoringTypeComplete means that group costs TestGroup.GroupScore (all the tests should be OK) TestGroupScoringTypeComplete TestGroupScoringType = iota + 1 // TestGroupScoringTypeEachTest means that group score = TestGroup.TestScore * (number of tests with OK) TestGroupScoringTypeEachTest @@ -44,12 +44,12 @@ const ( type TestGroup struct { Name string `json:"name" yaml:"name"` - FirstTest int `json:"FirstTest" yaml:"FirstTest"` - LastTest int `json:"LastTest" yaml:"LastTest"` + FirstTest uint64 `json:"FirstTest" yaml:"FirstTest"` + LastTest uint64 `json:"LastTest" yaml:"LastTest"` // TestScore meaningful only in case of TestGroupScoringTypeEachTest TestScore *float64 `json:"TestScore" yaml:"TestScore"` - // Score meaningful only in case of TestGroupScoringTypeComplete - Score *float64 `json:"Score" yaml:"Score"` + // GroupScore meaningful only in case of TestGroupScoringTypeComplete + GroupScore *float64 `json:"GroupScore" yaml:"GroupScore"` ScoringType TestGroupScoringType `json:"ScoringType" yaml:"ScoringType"` FeedbackType TestGroupFeedbackType `json:"FeedbackType" yaml:"FeedbackType"` RequiredGroupNames []string `json:"RequiredGroupNames" yaml:"RequiredGroupNames"` diff --git a/master/queue/jobgenerators/generator_test.go b/master/queue/jobgenerators/generator_test.go index 55c1074..70d9d76 100644 --- a/master/queue/jobgenerators/generator_test.go +++ b/master/queue/jobgenerators/generator_test.go @@ -11,23 +11,16 @@ import ( "testing_system/common/db/models" ) -func fixtureICPCProblem() *models.Problem { - return &models.Problem{ - ProblemType: models.ProblemTypeICPC, - TestsNumber: 10, - } -} - -func fixtureSubmission() *models.Submission { +func fixtureSubmission(ID uint) *models.Submission { submission := &models.Submission{} - submission.ID = 1 + submission.ID = ID return submission } func nextJob(t *testing.T, g Generator, SubmitID uint, jobType invokerconn.JobType, test uint64) *invokerconn.Job { job := g.NextJob() require.NotNil(t, job) - require.Equal(t, job.Type, jobType) + require.Equal(t, jobType, job.Type) require.Equal(t, SubmitID, job.SubmitID) require.Equal(t, test, job.Test) return job @@ -39,8 +32,16 @@ func noJobs(t *testing.T, g Generator) { } func TestICPCGenerator(t *testing.T) { + const fixtureICPCProblemTestsNumber = 10 + fixtureICPCProblem := func() *models.Problem { + return &models.Problem{ + ProblemType: models.ProblemTypeICPC, + TestsNumber: fixtureICPCProblemTestsNumber, + } + } + t.Run("Fail compilation", func(t *testing.T) { - problem, submission := fixtureICPCProblem(), fixtureSubmission() + problem, submission := fixtureICPCProblem(), fixtureSubmission(1) g, err := NewGenerator(problem, submission) require.Nil(t, err) job := nextJob(t, g, 1, invokerconn.CompileJob, 0) @@ -61,7 +62,7 @@ func TestICPCGenerator(t *testing.T) { t.Run("Straight tasks finishing", func(t *testing.T) { problem := fixtureICPCProblem() - submission := fixtureSubmission() + submission := fixtureSubmission(1) generator, err := NewGenerator(problem, submission) require.Nil(t, err) job := nextJob(t, generator, 1, invokerconn.CompileJob, 0) @@ -72,7 +73,7 @@ func TestICPCGenerator(t *testing.T) { }) require.Nil(t, sub) require.Nil(t, err) - for i := range 9 { + for i := range fixtureICPCProblemTestsNumber - 1 { job = nextJob(t, generator, 1, invokerconn.TestJob, uint64(i)+1) sub, err = generator.JobCompleted(&masterconn.InvokerJobResult{ JobID: job.ID, @@ -81,7 +82,7 @@ func TestICPCGenerator(t *testing.T) { require.Nil(t, sub) require.Nil(t, err) } - job = nextJob(t, generator, 1, invokerconn.TestJob, 10) + job = nextJob(t, generator, 1, invokerconn.TestJob, fixtureICPCProblemTestsNumber) sub, err = generator.JobCompleted(&masterconn.InvokerJobResult{ JobID: job.ID, Verdict: verdict.OK, @@ -100,7 +101,7 @@ func TestICPCGenerator(t *testing.T) { t.Run("Tasks finishing", func(t *testing.T) { prepare := func() (Generator, []string) { problem := fixtureICPCProblem() - submission := fixtureSubmission() + submission := fixtureSubmission(1) g, err := NewGenerator(problem, submission) require.Nil(t, err) job := nextJob(t, g, 1, invokerconn.CompileJob, 0) @@ -118,7 +119,7 @@ func TestICPCGenerator(t *testing.T) { return g, firstTwoJobIDs } finishOtherTests := func(g Generator) { - for i := 2; i < 9; i++ { + for i := 2; i < fixtureICPCProblemTestsNumber-1; i++ { job := nextJob(t, g, 1, invokerconn.TestJob, uint64(i)+1) sub, err := g.JobCompleted(&masterconn.InvokerJobResult{ JobID: job.ID, @@ -229,7 +230,7 @@ func TestICPCGenerator(t *testing.T) { }) t.Run("Finish same job twice", func(t *testing.T) { - problem, submission := fixtureICPCProblem(), fixtureSubmission() + problem, submission := fixtureICPCProblem(), fixtureSubmission(1) g, err := NewGenerator(problem, submission) require.Nil(t, err) job := nextJob(t, g, 1, invokerconn.CompileJob, 0) @@ -274,7 +275,7 @@ func TestIOIGenerator(t *testing.T) { }, TestsNumber: 1, }, - // type Complete, but Score is nil + // type Complete, but GroupScore is nil { ProblemType: models.ProblemTypeIOI, TestGroups: []models.TestGroup{ @@ -282,7 +283,7 @@ func TestIOIGenerator(t *testing.T) { Name: "name", FirstTest: 1, LastTest: 1, - Score: nil, + GroupScore: nil, ScoringType: models.TestGroupScoringTypeComplete, RequiredGroupNames: make([]string, 0), }, @@ -386,7 +387,7 @@ func TestIOIGenerator(t *testing.T) { FirstTest: 1, LastTest: 10, TestScore: nil, - Score: pointer.Float64(100), + GroupScore: pointer.Float64(100), ScoringType: models.TestGroupScoringTypeComplete, RequiredGroupNames: make([]string, 0), }, @@ -394,7 +395,7 @@ func TestIOIGenerator(t *testing.T) { } t.Run("Fail compilation", func(t *testing.T) { - submission := fixtureSubmission() + submission := fixtureSubmission(1) wasProblem := problemWithOneGroup g, err := NewGenerator(&problemWithOneGroup, submission) require.Nil(t, err) @@ -424,7 +425,7 @@ func TestIOIGenerator(t *testing.T) { t.Run("Straight task finishing", func(t *testing.T) { wasProblem := problemWithOneGroup - submission := fixtureSubmission() + submission := fixtureSubmission(1) generator, err := NewGenerator(&problemWithOneGroup, submission) require.Nil(t, err) job := nextJob(t, generator, 1, invokerconn.CompileJob, 0) @@ -480,7 +481,7 @@ func TestIOIGenerator(t *testing.T) { FirstTest: 1, LastTest: 1, TestScore: nil, - Score: pointer.Float64(50), + GroupScore: pointer.Float64(50), ScoringType: models.TestGroupScoringTypeComplete, RequiredGroupNames: make([]string, 0), }, @@ -489,7 +490,7 @@ func TestIOIGenerator(t *testing.T) { FirstTest: 2, LastTest: 2, TestScore: nil, - Score: pointer.Float64(50), + GroupScore: pointer.Float64(50), ScoringType: models.TestGroupScoringTypeComplete, RequiredGroupNames: []string{"group1"}, }, @@ -674,7 +675,7 @@ func TestIOIGenerator(t *testing.T) { FirstTest: 1, LastTest: 2, TestScore: nil, - Score: pointer.Float64(100), + GroupScore: pointer.Float64(100), ScoringType: models.TestGroupScoringTypeComplete, RequiredGroupNames: make([]string, 0), }, @@ -728,7 +729,7 @@ func TestIOIGenerator(t *testing.T) { }, }, } - g, err := NewIOIGenerator(&problem, fixtureSubmission()) + g, err := NewIOIGenerator(&problem, fixtureSubmission(1)) require.NoError(t, err) job := nextJob(t, g, 1, invokerconn.CompileJob, 0) require.NotNil(t, job) @@ -789,7 +790,7 @@ func TestIOIGenerator(t *testing.T) { Name: "group1", FirstTest: 1, LastTest: 3, - Score: pointer.Float64(20), + GroupScore: pointer.Float64(20), ScoringType: models.TestGroupScoringTypeMin, RequiredGroupNames: make([]string, 0), }, @@ -799,7 +800,7 @@ func TestIOIGenerator(t *testing.T) { t.Run("WA in the middle of the group", func(t *testing.T) { problem := prepare() - g, err := NewIOIGenerator(&problem, fixtureSubmission()) + g, err := NewIOIGenerator(&problem, fixtureSubmission(1)) require.NoError(t, err) job := nextJob(t, g, 1, invokerconn.CompileJob, 0) require.NotNil(t, job) @@ -851,7 +852,7 @@ func TestIOIGenerator(t *testing.T) { t.Run("PT in the middle of the group", func(t *testing.T) { problem := prepare() - g, err := NewIOIGenerator(&problem, fixtureSubmission()) + g, err := NewIOIGenerator(&problem, fixtureSubmission(1)) require.NoError(t, err) job := nextJob(t, g, 1, invokerconn.CompileJob, 0) require.NotNil(t, job) @@ -904,7 +905,7 @@ func TestIOIGenerator(t *testing.T) { t.Run("no fails", func(t *testing.T) { problem := prepare() - g, err := NewIOIGenerator(&problem, fixtureSubmission()) + g, err := NewIOIGenerator(&problem, fixtureSubmission(1)) require.NoError(t, err) job := nextJob(t, g, 1, invokerconn.CompileJob, 0) require.NotNil(t, job) diff --git a/master/queue/jobgenerators/ioi_generator.go b/master/queue/jobgenerators/ioi_generator.go index 2025c93..888a93b 100644 --- a/master/queue/jobgenerators/ioi_generator.go +++ b/master/queue/jobgenerators/ioi_generator.go @@ -15,74 +15,83 @@ import ( ) type IOIGenerator struct { - id string - mutex sync.Mutex - submission *models.Submission - problem *models.Problem - state generatorState - givenJobs map[string]*invokerconn.Job + id string + mutex sync.Mutex + submission *models.Submission + problem *models.Problem + state generatorState + givenJobs map[string]*invokerconn.Job + groupNameToGroupInfo map[string]models.TestGroup + groupNameToInternalInfo map[string]*internalGroupInfo + testNumberToGroupName map[uint64]string + internalTestResults []*internalTestResult - // groupNameToOrigGroup should be set once at the creating of the generator and should not be changed further - groupNameToOrigGroup map[string]*models.TestGroup - // origGroupNamesToBeGiven should be set once at the creating of the generator and should not be changed further - origGroupNamesToBeGiven []string - - groupNameToGroupInfo map[string]*internalGroupInfo - groupNamesToBeGiven []string - testNumberToGroupName map[uint64]string + // firstNotCompletedTest = the longest prefix of the tests, for which we know verdict + firstNotCompletedTest uint64 + // firstNotCompletedGroup = the longest prefix of the groups, for which we know verdict + firstNotCompletedGroup uint64 + firstUnseenTest uint64 } -type groupState int +type internalGroupState int + +const ( + groupRunning internalGroupState = iota + groupFailed // if the group isn't failed and completed, it will still have groupRunning state +) + +type internalTestState int const ( - onlyOK groupState = iota - okOrPT - hasFails + testNotGiven internalTestState = iota + testRunning + testFinished ) type internalGroupInfo struct { - group models.TestGroup - minPoints float64 - sumPoints float64 - testsPasses int - groupState groupState + state internalGroupState + shouldSkipTests bool } -type problemWalkthroughResults struct { - order []string - used map[string]int - err error +type internalTestResult struct { + result *models.TestResult + state internalTestState } -func (i *IOIGenerator) walkThroughGroups(groupInfo *internalGroupInfo, result *problemWalkthroughResults) { - result.used[groupInfo.group.Name] = 1 - for _, requiredGroupName := range groupInfo.group.RequiredGroupNames { - requiredGroupInfo := i.groupNameToGroupInfo[requiredGroupName] - if requiredGroupInfo.group.ScoringType == models.TestGroupScoringTypeEachTest || - requiredGroupInfo.group.ScoringType == models.TestGroupScoringTypeMin { - result.err = fmt.Errorf("group can't depend on group with scoring type each test or min") - } else if val, ok := result.used[requiredGroupInfo.group.Name]; !ok { - i.walkThroughGroups(requiredGroupInfo, result) - } else if val == 1 { - result.err = fmt.Errorf("cycle detected in dependencies") +func (i *IOIGenerator) checkIfGroupsDependenciesOK(problem *models.Problem) error { + if !slices.IsSortedFunc(problem.TestGroups, func(a, b models.TestGroup) int { + return int(a.FirstTest) - int(b.FirstTest) + }) { + return fmt.Errorf("groups are not sorted by tests") + } + // each test should be in exactly one group, and each group should depend on the earlier ones + lastTest := uint64(0) + encounteredGroups := make(map[string]struct{}) + for _, group := range problem.TestGroups { + if group.FirstTest != lastTest+1 { + return fmt.Errorf("at least one test is not in exactly one group") } - } - result.order = append(result.order, groupInfo.group.Name) - result.used[groupInfo.group.Name] = 2 -} - -func (i *IOIGenerator) getWalkthroughResults() *problemWalkthroughResults { - result := &problemWalkthroughResults{ - order: make([]string, 0), - used: make(map[string]int), - err: nil, - } - for _, groupInfo := range i.groupNameToGroupInfo { - if _, ok := result.used[groupInfo.group.Name]; !ok { - i.walkThroughGroups(groupInfo, result) + lastTest = group.LastTest + if _, ok := encounteredGroups[group.Name]; ok { + return fmt.Errorf("duplicate test group: %s", group.Name) + } + for _, requiredGroupName := range group.RequiredGroupNames { + if _, ok := encounteredGroups[requiredGroupName]; !ok { + return fmt.Errorf("missing required group (maybe it is later) %v for group %v", + requiredGroupName, group.Name) + } + requiredGroupInfo := i.groupNameToGroupInfo[requiredGroupName] + if requiredGroupInfo.ScoringType != models.TestGroupScoringTypeComplete { + return fmt.Errorf("group %v depends on group %v with scoringType=%v", + group.Name, requiredGroupInfo.Name, requiredGroupInfo.ScoringType) + } } + encounteredGroups[group.Name] = struct{}{} + } + if lastTest != problem.TestsNumber { + return fmt.Errorf("at least one test is not in exactly one group") } - return result + return nil } func (i *IOIGenerator) prepareGenerator() error { @@ -90,30 +99,12 @@ func (i *IOIGenerator) prepareGenerator() error { if problem.ProblemType != models.ProblemTypeIOI { return fmt.Errorf("problem %v is not an IOI problem", problem.ID) } - // must copy groups in order to not break the problem - for _, group := range problem.TestGroups { - if _, ok := i.groupNameToGroupInfo[group.Name]; ok { - return fmt.Errorf("group %v presented twice in problem", problem.ID) - } - i.groupNameToGroupInfo[group.Name] = &internalGroupInfo{ - group: group, - minPoints: math.Inf(1), - sumPoints: 0, - testsPasses: 0, - groupState: onlyOK, - } - group1 := group - i.groupNameToOrigGroup[group1.Name] = &group1 - for testNumber := group.FirstTest - 1; testNumber < group.LastTest; testNumber++ { - i.testNumberToGroupName[uint64(testNumber)] = group.Name - } - } // each group with TestGroupScoringTypeEachTest must have TestScore for _, group := range problem.TestGroups { switch group.ScoringType { case models.TestGroupScoringTypeComplete, models.TestGroupScoringTypeMin: - if group.Score == nil { - return fmt.Errorf("group %v in problem %v doesn't have Score", group.Name, problem.ID) + if group.GroupScore == nil { + return fmt.Errorf("group %v in problem %v doesn't have GroupScore", group.Name, problem.ID) } case models.TestGroupScoringTypeEachTest: if group.TestScore == nil { @@ -122,133 +113,23 @@ func (i *IOIGenerator) prepareGenerator() error { default: return fmt.Errorf("unknown TestGroupScoringType %v", group.ScoringType) } - } - { - // simple check that each test is in exactly one group - for testNumber := range i.problem.TestsNumber { - cnt := 0 - for _, group := range i.problem.TestGroups { - if uint64(group.FirstTest) <= testNumber+1 && testNumber+1 <= uint64(group.LastTest) { - cnt += 1 - } - } - if cnt != 1 { - return fmt.Errorf("test %v is presented in %v groups in problem %v", testNumber, problem.ID, cnt) - } - } - } - { - // check for cycles and build order of groups - result := i.getWalkthroughResults() - if result.err != nil { - return result.err - } - i.groupNamesToBeGiven = result.order - i.origGroupNamesToBeGiven = result.order - } - { - testResults := make([]models.TestResult, 0, problem.TestsNumber) - for testNumber := range problem.TestsNumber { - testResults = append(testResults, models.TestResult{ - TestNumber: testNumber + 1, - Verdict: verdict.SK, - Time: 0, - Memory: 0, - }) - } - i.submission.TestResults = testResults - } - return nil -} - -// finalizeResultsAfterTestJobCompleted must be done with acquired mutex and only ONCE (in case of no CE) -func (i *IOIGenerator) finalizeResultsAfterTestJobCompleted() { - totalScore := 0.0 - i.submission.Verdict = verdict.OK - for _, groupName := range i.origGroupNamesToBeGiven { - groupInfo := i.groupNameToGroupInfo[groupName] - origGroup := i.groupNameToOrigGroup[groupName] - // if at least one required group is failed, then current group is fully SK - haveFail := false - for _, requiredGroupName := range origGroup.RequiredGroupNames { - if i.groupNameToGroupInfo[requiredGroupName].groupState == hasFails { - haveFail = true - } - } - if haveFail { - for testNumber := origGroup.FirstTest - 1; testNumber < origGroup.LastTest; testNumber++ { - i.submission.TestResults[testNumber].Verdict = verdict.SK - i.submission.TestResults[testNumber].Points = pointer.Float64(0) - } - i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ - GroupName: groupName, - Points: 0.0, - Passed: false, + for testNumber := group.FirstTest; testNumber <= group.LastTest; testNumber++ { + i.testNumberToGroupName[testNumber-1] = group.Name + i.internalTestResults = append(i.internalTestResults, &internalTestResult{ + result: &models.TestResult{ + TestNumber: testNumber, + Verdict: verdict.RU, + }, + state: testNotGiven, }) - continue - } - // otherwise calculate score for this group and set appropriate verdicts - if groupInfo.groupState != onlyOK { - i.submission.Verdict = verdict.PT - } - curPoints := 0.0 - setSKVerdicts := func(notAcceptableVerdicts ...verdict.Verdict) { - // should set SK for all the tests after the first one without OK - setSK := false - for testNumber := origGroup.FirstTest - 1; testNumber < origGroup.LastTest; testNumber++ { - if !slices.Contains(notAcceptableVerdicts, i.submission.TestResults[testNumber].Verdict) { - setSK = true - continue - } - if setSK { - i.submission.TestResults[testNumber].Verdict = verdict.SK - } - } - } - switch origGroup.ScoringType { - case models.TestGroupScoringTypeComplete: - if groupInfo.groupState == onlyOK { - curPoints = *origGroup.Score - } else { - setSKVerdicts(verdict.OK) - } - case models.TestGroupScoringTypeEachTest: - curPoints = groupInfo.sumPoints - case models.TestGroupScoringTypeMin: - if groupInfo.groupState == hasFails { - setSKVerdicts(verdict.OK, verdict.PT) - curPoints = 0.0 - } else { - if groupInfo.minPoints != math.Inf(1) { - curPoints = groupInfo.minPoints - } - } } - totalScore += curPoints - i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ - GroupName: groupName, - Points: curPoints, - Passed: groupInfo.groupState == onlyOK, - }) - } - i.submission.Score = totalScore -} - -// finalizeResultsAfterCompileJobFailed must be done with acquired mutex and only ONCE (in case of CE) -func (i *IOIGenerator) finalizeResultsAfterCompileJobFailed() { - i.submission.Verdict = verdict.CE - for _, groupName := range i.origGroupNamesToBeGiven { - origGroup := i.groupNameToOrigGroup[groupName] - for testNumber := origGroup.FirstTest - 1; testNumber < origGroup.LastTest; testNumber++ { - i.submission.TestResults[testNumber].Verdict = verdict.SK + i.groupNameToGroupInfo[group.Name] = group + i.groupNameToInternalInfo[group.Name] = &internalGroupInfo{ + state: groupRunning, + shouldSkipTests: false, } - i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ - GroupName: groupName, - Points: 0.0, - Passed: false, - }) } - i.submission.Score = 0.0 + return i.checkIfGroupsDependenciesOK(problem) } func (i *IOIGenerator) ID() string { @@ -258,7 +139,7 @@ func (i *IOIGenerator) ID() string { func (i *IOIGenerator) NextJob() *invokerconn.Job { i.mutex.Lock() defer i.mutex.Unlock() - if i.state == compilationFinished && len(i.groupNamesToBeGiven) == 0 { + if i.state == compilationFinished && i.firstUnseenTest > i.problem.TestsNumber { return nil } if i.state == compilationStarted { @@ -279,20 +160,118 @@ func (i *IOIGenerator) NextJob() *invokerconn.Job { return job } job.Type = invokerconn.TestJob - for len(i.groupNameToGroupInfo) > 0 { - groupName := i.groupNamesToBeGiven[0] - groupInfo := i.groupNameToGroupInfo[groupName] - job.Test = uint64(groupInfo.group.FirstTest) - groupInfo.group.FirstTest++ - if groupInfo.group.FirstTest > groupInfo.group.LastTest { - i.groupNamesToBeGiven = i.groupNamesToBeGiven[1:] + for i.firstUnseenTest < i.problem.TestsNumber { + groupName := i.testNumberToGroupName[i.firstUnseenTest] + groupInfo := i.groupNameToInternalInfo[groupName] + if groupInfo.shouldSkipTests { + i.firstUnseenTest++ + continue } + i.internalTestResults[i.firstUnseenTest].state = testRunning + i.firstUnseenTest++ + job.Test = i.firstUnseenTest i.givenJobs[job.ID] = job return job } return nil } +func isTestFailed(testGroupInfo models.TestGroup, testVerdict verdict.Verdict) bool { + switch testGroupInfo.ScoringType { + case models.TestGroupScoringTypeComplete: + return testVerdict != verdict.OK + case models.TestGroupScoringTypeEachTest: + return testVerdict != verdict.OK + case models.TestGroupScoringTypeMin: + return testVerdict != verdict.OK + default: + panic(fmt.Sprintf("unknown testGroupInfo.ScoringType %v", testGroupInfo.ScoringType)) + } +} + +func shouldSkipNewTests(testGroupInfo models.TestGroup, testVerdict verdict.Verdict) bool { + switch testGroupInfo.ScoringType { + case models.TestGroupScoringTypeComplete: + return testVerdict != verdict.OK + case models.TestGroupScoringTypeEachTest: + return false + case models.TestGroupScoringTypeMin: + return testVerdict != verdict.OK && testVerdict != verdict.PT + default: + panic(fmt.Sprintf("unknown testGroupInfo.ScoringType %v", testGroupInfo.ScoringType)) + } +} + +// increaseCompletedTestsAndGroups must be done with acquired mutex +func (i *IOIGenerator) increaseCompletedTestsAndGroups() { +TestsLoop: + for i.firstNotCompletedTest != i.problem.TestsNumber { + testGroupName := i.testNumberToGroupName[i.firstNotCompletedTest] + testGroupInfo := i.groupNameToGroupInfo[testGroupName] + testInternalGroupInfo := i.groupNameToInternalInfo[testGroupName] + switch i.internalTestResults[i.firstNotCompletedTest].state { + case testNotGiven: + if !testInternalGroupInfo.shouldSkipTests { + break TestsLoop + } + case testRunning: + break TestsLoop + case testFinished: + } + i.submission.TestResults = append(i.submission.TestResults, + *i.internalTestResults[i.firstNotCompletedTest].result) + if !testInternalGroupInfo.shouldSkipTests { + if shouldSkipNewTests(testGroupInfo, i.submission.TestResults[i.firstNotCompletedTest].Verdict) { + testInternalGroupInfo.shouldSkipTests = true + // update shouldSkipTests + for _, group := range i.problem.TestGroups { + for _, requiredGroupName := range group.RequiredGroupNames { + if i.groupNameToInternalInfo[requiredGroupName].shouldSkipTests { + i.groupNameToInternalInfo[group.Name].shouldSkipTests = true + } + } + } + } + } else { + i.submission.TestResults[i.firstNotCompletedTest].Verdict = verdict.SK + } + if i.firstNotCompletedTest+1 == testGroupInfo.LastTest { + score := 0.0 + switch testGroupInfo.ScoringType { + case models.TestGroupScoringTypeComplete: + if testInternalGroupInfo.state != groupFailed { + score = *testGroupInfo.GroupScore + } + case models.TestGroupScoringTypeEachTest: + for testNumber := testGroupInfo.FirstTest - 1; testNumber < testGroupInfo.LastTest; testNumber++ { + if testScore := i.submission.TestResults[testNumber].Points; testScore != nil { + score += *testScore + } + } + case models.TestGroupScoringTypeMin: + score = math.Inf(+1) + for testNumber := testGroupInfo.FirstTest - 1; testNumber < testGroupInfo.LastTest; testNumber++ { + if testScore := i.submission.TestResults[testNumber].Points; testScore != nil { + score = min(score, *testScore) + } else { + score = 0.0 + } + } + if score == math.Inf(+1) { + score = 0 + } + } + i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ + GroupName: testGroupName, + Points: score, + Passed: testInternalGroupInfo.state != groupFailed, + }) + i.firstNotCompletedGroup++ + } + i.firstNotCompletedTest++ + } +} + // compileJobCompleted must be done with acquired mutex func (i *IOIGenerator) compileJobCompleted(job *invokerconn.Job, result *masterconn.InvokerJobResult) (*models.Submission, error) { if job.Type != invokerconn.CompileJob { @@ -303,127 +282,124 @@ func (i *IOIGenerator) compileJobCompleted(job *invokerconn.Job, result *masterc i.state = compilationFinished return nil, nil case verdict.CE: - i.finalizeResultsAfterCompileJobFailed() + i.submission.Verdict = verdict.CE + for _, group := range i.problem.TestGroups { + i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ + GroupName: group.Name, + Points: 0, + Passed: false, + }) + } + for t := range i.problem.TestsNumber { + i.submission.TestResults = append(i.submission.TestResults, models.TestResult{ + TestNumber: t + 1, + Verdict: verdict.SK, + }) + } return i.submission, nil default: return nil, fmt.Errorf("unknown verdict for compilation completed: %v", result.Verdict) } } -// testJobCompleted must be done with acquired mutex -func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn.InvokerJobResult) (*models.Submission, error) { - if job.Type != invokerconn.TestJob { - return nil, fmt.Errorf("job type %v is not test job", job.ID) - } - groupName := i.testNumberToGroupName[job.Test-1] - groupInfo := i.groupNameToGroupInfo[groupName] - - // a lot of checks for fucking IOI problems - setResultCF := func(checkerScore *float64, problemId uint, group string) { - if len(result.Error) > 0 { - result.Error += "; " - } - result.Error += fmt.Sprintf( - "checker returned score=%v, but testScore=%v and score=%v in problemId=%v, group=%v, type=%v", - checkerScore, groupInfo.group.TestScore, groupInfo.group.Score, - problemId, group, groupInfo.group.ScoringType) - result.Verdict = verdict.CF - result.Points = pointer.Float64(0) - } - if groupInfo.group.ScoringType == models.TestGroupScoringTypeEachTest { +func populateTestJobResult(groupInfo models.TestGroup, result *masterconn.InvokerJobResult) error { + switch groupInfo.ScoringType { + case models.TestGroupScoringTypeComplete: + case models.TestGroupScoringTypeEachTest: if result.Points == nil { if result.Verdict == verdict.OK { - result.Points = groupInfo.group.TestScore + result.Points = groupInfo.TestScore } else if result.Verdict == verdict.PT { - setResultCF(result.Points, i.problem.ID, groupName) + return fmt.Errorf("checker returned nil points, but verdict=%v", result.Verdict) + } else { + result.Points = pointer.Float64(0) } - } else if *result.Points > *groupInfo.group.TestScore { - setResultCF(result.Points, i.problem.ID, groupName) } - } else if groupInfo.group.ScoringType == models.TestGroupScoringTypeMin { + case models.TestGroupScoringTypeMin: if result.Points == nil { if result.Verdict == verdict.OK { - result.Points = groupInfo.group.Score + result.Points = groupInfo.GroupScore + return nil } else if result.Verdict == verdict.PT { - setResultCF(result.Points, i.problem.ID, groupName) - } - } else if *result.Points > *groupInfo.group.Score { - setResultCF(result.Points, i.problem.ID, groupName) - } - } else if groupInfo.group.ScoringType == models.TestGroupScoringTypeComplete { - if result.Verdict == verdict.PT { - result.Verdict = verdict.CF - if len(result.Error) > 0 { - result.Error += "; " + return fmt.Errorf("checker returned nil points, but verdict=%v", result.Verdict) + } else { + result.Points = pointer.Float64(0) } - result.Error += fmt.Sprintf("checker returned PT on test=%v in problemId=%v, group=%v, type=%v", - job.Test, i.problem.ID, groupName, groupInfo.group.ScoringType) } + default: + panic(fmt.Sprintf("unknown group scoring type: %v", groupInfo.ScoringType)) } + return nil +} - i.submission.TestResults[job.Test-1].Verdict = result.Verdict - if result.Points != nil { - groupInfo.minPoints = min(groupInfo.minPoints, *result.Points) - groupInfo.sumPoints += *result.Points - } - if result.Statistics != nil { - i.submission.TestResults[job.Test-1].Time = result.Statistics.Time - i.submission.TestResults[job.Test-1].Memory = result.Statistics.Memory +// testJobCompleted must be done with acquired mutex +func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn.InvokerJobResult) (*models.Submission, error) { + if job.Type != invokerconn.TestJob { + return nil, fmt.Errorf("job type %v is not test job", job.ID) } - - switch result.Verdict { - case verdict.PT: - if groupInfo.groupState != hasFails { - groupInfo.groupState = okOrPT + test := job.Test - 1 + testGroupName := i.testNumberToGroupName[test] + testGroupInfo := i.groupNameToGroupInfo[testGroupName] + testInternalGroupInfo := i.groupNameToInternalInfo[testGroupName] + // move info from result to internal state + if err := populateTestJobResult(testGroupInfo, result); err != nil { + result.Verdict = verdict.CF + if result.Points != nil { + result.Points = pointer.Float64(0) } - fallthrough - case verdict.OK: - if groupInfo.group.ScoringType == models.TestGroupScoringTypeEachTest || - groupInfo.group.ScoringType == models.TestGroupScoringTypeMin { - i.submission.TestResults[job.Test-1].Points = result.Points + if result.Error != "" { + result.Error += "; " } - case verdict.CF: - i.submission.TestResults[job.Test-1].Error = result.Error - fallthrough - case verdict.WA, verdict.RT, verdict.ML, verdict.TL, verdict.WL, verdict.SE: - groupInfo.groupState = hasFails - default: - return nil, fmt.Errorf("unknown verdict for test %v in job %v", job.Test, job.ID) + result.Error += err.Error() } + i.internalTestResults[test].result.Points = result.Points + i.internalTestResults[test].result.Verdict = result.Verdict + i.internalTestResults[test].result.Error = result.Error + if result.Statistics != nil { + i.internalTestResults[test].result.Time = result.Statistics.Time + i.internalTestResults[test].result.Memory = result.Statistics.Memory + } + i.internalTestResults[test].state = testFinished - /* - scenario: - group1: any scoring type, tests: 1-1 - group2: any scoring type, tests: 2-2 - 1. NextJob returns the test from the first group - 2. WA it - 3. It is expected that we already know testing result - so we have to go through all the groups and check for their required groups if they are not passed already - */ - { - newGroupNamesToBeGiven := make([]string, 0) - for _, groupName = range i.groupNamesToBeGiven { - groupInfo = i.groupNameToGroupInfo[groupName] - shouldTest := true - for _, requiredGroupName := range groupInfo.group.RequiredGroupNames { - if i.groupNameToGroupInfo[requiredGroupName].groupState != onlyOK { - shouldTest = false + if isTestFailed(testGroupInfo, result.Verdict) { + testInternalGroupInfo.state = groupFailed + for _, group := range i.problem.TestGroups { + for _, requiredGroupName := range group.RequiredGroupNames { + if i.groupNameToInternalInfo[requiredGroupName].state == groupFailed { + i.groupNameToInternalInfo[group.Name].state = groupFailed break } } - // this is the only case, since we assume that checker can not return PT in type complete - if groupInfo.groupState == hasFails { - shouldTest = false - } - if shouldTest { - newGroupNamesToBeGiven = append(newGroupNamesToBeGiven, groupName) - } } - i.groupNamesToBeGiven = newGroupNamesToBeGiven } + i.increaseCompletedTestsAndGroups() + if i.firstNotCompletedTest == i.problem.TestsNumber && len(i.givenJobs) == 0 { + if len(i.submission.GroupResults) != len(i.problem.TestGroups) { + panic("for some reason \"len(i.submission.GroupResults) != len(i.problem.TestGroups)\"") + } + if len(i.submission.TestResults) != int(i.problem.TestsNumber) { + panic("for some reason \"len(i.submission.TestResults) != int(i.problem.TestsNumber)\"") + } + for _, groupResult := range i.submission.GroupResults { + i.submission.Score += groupResult.Points + } - if len(i.givenJobs) == 0 && len(i.groupNamesToBeGiven) == 0 { - i.finalizeResultsAfterTestJobCompleted() + haveSkipped := false + haveVerdict := false + for _, testResult := range i.submission.TestResults { + if testResult.Verdict != verdict.OK && testResult.Verdict != verdict.SK { + i.submission.Verdict = verdict.PT + haveVerdict = true + } else if testResult.Verdict == verdict.SK { + haveSkipped = true + } + } + if !haveVerdict { + if haveSkipped { + panic("problem does not have verdict, but have skipped tests") + } + i.submission.Verdict = verdict.OK + } return i.submission, nil } return nil, nil @@ -434,7 +410,7 @@ func (i *IOIGenerator) JobCompleted(jobResult *masterconn.InvokerJobResult) (*mo defer i.mutex.Unlock() job, ok := i.givenJobs[jobResult.JobID] if !ok { - return nil, fmt.Errorf("job %s not exist", jobResult.JobID) + return nil, fmt.Errorf("job %s does not exist", jobResult.JobID) } delete(i.givenJobs, jobResult.JobID) switch job.Type { @@ -453,17 +429,18 @@ func NewIOIGenerator(problem *models.Problem, submission *models.Submission) (Ge logger.Panic("Can't generate generator id: %w", err) } generator := &IOIGenerator{ - id: id.String(), - submission: submission, - problem: problem, - state: compilationNotStarted, - givenJobs: make(map[string]*invokerconn.Job), - groupNameToGroupInfo: make(map[string]*internalGroupInfo), - // these fields will be filled in prepareGenerator - groupNameToOrigGroup: make(map[string]*models.TestGroup), - groupNamesToBeGiven: nil, - origGroupNamesToBeGiven: nil, + id: id.String(), + submission: submission, + problem: problem, + state: compilationNotStarted, + givenJobs: make(map[string]*invokerconn.Job), + groupNameToGroupInfo: make(map[string]models.TestGroup), + groupNameToInternalInfo: make(map[string]*internalGroupInfo), testNumberToGroupName: make(map[uint64]string), + internalTestResults: make([]*internalTestResult, 0), + firstNotCompletedTest: 0, + firstNotCompletedGroup: 0, + firstUnseenTest: 0, } if err = generator.prepareGenerator(); err != nil { return nil, err From 4a602a92510601c7e72ae207616f89ee26ff1276 Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Thu, 1 May 2025 13:14:46 +0300 Subject: [PATCH 05/10] small fixes --- common/db/models/problem.go | 2 +- common/db/models/submission.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/db/models/problem.go b/common/db/models/problem.go index 736f9c9..b1d41e2 100644 --- a/common/db/models/problem.go +++ b/common/db/models/problem.go @@ -62,7 +62,7 @@ func (t TestGroup) Value() (driver.Value, error) { func (t *TestGroup) Scan(value interface{}) error { bytes, ok := value.([]byte) if !ok { - return errors.New("type assertion to []byte failed") + return errors.New("type assertion to []byte failed while scanning TestGroup") } return json.Unmarshal(bytes, t) } diff --git a/common/db/models/submission.go b/common/db/models/submission.go index 77eef27..1a24569 100644 --- a/common/db/models/submission.go +++ b/common/db/models/submission.go @@ -28,7 +28,7 @@ func (t TestResults) Value() (driver.Value, error) { func (t *TestResults) Scan(value interface{}) error { bytes, ok := value.([]byte) if !ok { - return errors.New("type assertion to []byte failed") + return errors.New("type assertion to []byte failed while scanning TestResults") } return json.Unmarshal(bytes, t) } @@ -59,7 +59,7 @@ func (t GroupResults) Value() (driver.Value, error) { func (t *GroupResults) Scan(value interface{}) error { bytes, ok := value.([]byte) if !ok { - return errors.New("type assertion to []byte failed") + return errors.New("type assertion to []byte failed while scanning GroupResults") } return json.Unmarshal(bytes, t) } From 60fa9ad890fd31e7f3e4c642b6e1e940aff68db8 Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Thu, 1 May 2025 13:20:17 +0300 Subject: [PATCH 06/10] tmp --- common/db/models/problem.go | 56 +++++++++++-------------------------- 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/common/db/models/problem.go b/common/db/models/problem.go index b1d41e2..feafb42 100644 --- a/common/db/models/problem.go +++ b/common/db/models/problem.go @@ -22,7 +22,7 @@ const ( ProblemTypeIOI ) const ( - // TestGroupScoringTypeComplete means that group costs TestGroup.GroupScore (all the tests should be OK) + // TestGroupScoringTypeComplete means that group costs TestGroup.Score (all the tests should be OK) TestGroupScoringTypeComplete TestGroupScoringType = iota + 1 // TestGroupScoringTypeEachTest means that group score = TestGroup.TestScore * (number of tests with OK) TestGroupScoringTypeEachTest @@ -44,15 +44,15 @@ const ( type TestGroup struct { Name string `json:"name" yaml:"name"` - FirstTest uint64 `json:"FirstTest" yaml:"FirstTest"` - LastTest uint64 `json:"LastTest" yaml:"LastTest"` + FirstTest int `json:"first_test" yaml:"first_test"` + LastTest int `json:"last_test" yaml:"last_test"` // TestScore meaningful only in case of TestGroupScoringTypeEachTest - TestScore *float64 `json:"TestScore" yaml:"TestScore"` - // GroupScore meaningful only in case of TestGroupScoringTypeComplete - GroupScore *float64 `json:"GroupScore" yaml:"GroupScore"` - ScoringType TestGroupScoringType `json:"ScoringType" yaml:"ScoringType"` - FeedbackType TestGroupFeedbackType `json:"FeedbackType" yaml:"FeedbackType"` - RequiredGroupNames []string `json:"RequiredGroupNames" yaml:"RequiredGroupNames"` + TestScore *float64 `json:"test_score" yaml:"test_score"` + // Score meaningful only in case of TestGroupScoringTypeComplete + Score *float64 `json:"score" yaml:"score"` + ScoringType TestGroupScoringType `json:"scoring_type" yaml:"scoring_type"` + FeedbackType TestGroupFeedbackType `json:"feedback_type" yaml:"feedback_type"` + RequiredGroupNames []string `json:"required_groups" yaml:"required_groups"` } func (t TestGroup) Value() (driver.Value, error) { @@ -62,7 +62,7 @@ func (t TestGroup) Value() (driver.Value, error) { func (t *TestGroup) Scan(value interface{}) error { bytes, ok := value.([]byte) if !ok { - return errors.New("type assertion to []byte failed while scanning TestGroup") + return errors.New("type assertion to []byte failed") } return json.Unmarshal(bytes, t) } @@ -80,39 +80,17 @@ func (t TestGroup) GormDBDataType(db *gorm.DB, field *schema.Field) string { type Problem struct { gorm.Model -<<<<<<< HEAD - ProblemType ProblemType `yaml:"ProblemType"` - - TimeLimit customfields.Time `yaml:"TimeLimit"` - MemoryLimit customfields.Memory `yaml:"MemoryLimit"` - - TestsNumber uint64 `yaml:"TestsNumber"` + ProblemType ProblemType // TestGroups ignored for ICPC problems TestGroups []TestGroup -======= - ProblemType ProblemType ->>>>>>> f1bf2b0 (fixes and tests) - // WallTimeLimit specifies maximum execution and wait time. - // By default, it is max(5s, TimeLimit * 2) - WallTimeLimit *customfields.Time `yaml:"WallTimeLimit,omitempty"` + TimeLimit customfields.Time + MemoryLimit customfields.Memory + WallTimeLimit *customfields.Time -<<<<<<< HEAD - // MaxOpenFiles specifies the maximum number of files, opened by testing system. - // By default, it is 64 - MaxOpenFiles *uint64 `yaml:"MaxOpenFiles,omitempty"` -======= TestsNumber uint64 - // TestGroups ignored for ICPC problems - TestGroups []TestGroup ->>>>>>> f1bf2b0 (fixes and tests) - - // MaxThreads specifies the maximum number of threads and/or processes; - // By default, it is a single thread - // If MaxThreads equals to -1, any number of threads allowed - MaxThreads *int64 `yaml:"MaxThreads,omitempty"` - // MaxOutputSize specifies maximum output in EACH file. - // By default, it is 1g - MaxOutputSize *customfields.Memory `yaml:"MaxOutputSize,omitempty"` + MaxOpenFiles *uint64 + MaxThreads *uint64 + MaxOutputSize *customfields.Memory } From 40dc6f8857392170f51066b817d0a82f7a908514 Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Thu, 1 May 2025 13:28:03 +0300 Subject: [PATCH 07/10] try to resolve conflicts --- common/db/models/models_test.go | 4 +-- common/db/models/problem.go | 58 ++++++++++++++++++++------------- common/db/models/submission.go | 6 ++-- go.mod | 2 +- 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/common/db/models/models_test.go b/common/db/models/models_test.go index 92c6ad4..054dbb7 100644 --- a/common/db/models/models_test.go +++ b/common/db/models/models_test.go @@ -15,8 +15,8 @@ import ( func fixtureDb(t *testing.T) *gorm.DB { db, _ := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) - err := db.AutoMigrate(&Submission{}) - assert.NoError(t, err) + assert.NoError(t, db.AutoMigrate(&Submission{})) + assert.NoError(t, db.AutoMigrate(&Problem{})) return db } diff --git a/common/db/models/problem.go b/common/db/models/problem.go index feafb42..a6b56a8 100644 --- a/common/db/models/problem.go +++ b/common/db/models/problem.go @@ -22,7 +22,7 @@ const ( ProblemTypeIOI ) const ( - // TestGroupScoringTypeComplete means that group costs TestGroup.Score (all the tests should be OK) + // TestGroupScoringTypeComplete means that group costs TestGroup.GroupScore (all the tests should be OK) TestGroupScoringTypeComplete TestGroupScoringType = iota + 1 // TestGroupScoringTypeEachTest means that group score = TestGroup.TestScore * (number of tests with OK) TestGroupScoringTypeEachTest @@ -44,30 +44,32 @@ const ( type TestGroup struct { Name string `json:"name" yaml:"name"` - FirstTest int `json:"first_test" yaml:"first_test"` - LastTest int `json:"last_test" yaml:"last_test"` + FirstTest uint64 `json:"FirstTest" yaml:"FirstTest"` + LastTest uint64 `json:"LastTest" yaml:"LastTest"` // TestScore meaningful only in case of TestGroupScoringTypeEachTest - TestScore *float64 `json:"test_score" yaml:"test_score"` - // Score meaningful only in case of TestGroupScoringTypeComplete - Score *float64 `json:"score" yaml:"score"` - ScoringType TestGroupScoringType `json:"scoring_type" yaml:"scoring_type"` - FeedbackType TestGroupFeedbackType `json:"feedback_type" yaml:"feedback_type"` - RequiredGroupNames []string `json:"required_groups" yaml:"required_groups"` + TestScore *float64 `json:"TestScore" yaml:"TestScore"` + // GroupScore meaningful only in case of TestGroupScoringTypeComplete + GroupScore *float64 `json:"GroupScore" yaml:"GroupScore"` + ScoringType TestGroupScoringType `json:"ScoringType" yaml:"ScoringType"` + FeedbackType TestGroupFeedbackType `json:"FeedbackType" yaml:"FeedbackType"` + RequiredGroupNames []string `json:"RequiredGroupNames" yaml:"RequiredGroupNames"` } -func (t TestGroup) Value() (driver.Value, error) { +type TestGroups []TestGroup + +func (t TestGroups) Value() (driver.Value, error) { return json.Marshal(t) } -func (t *TestGroup) Scan(value interface{}) error { +func (t *TestGroups) Scan(value interface{}) error { bytes, ok := value.([]byte) if !ok { - return errors.New("type assertion to []byte failed") + return errors.New("type assertion to []byte failed while scanning TestGroups") } return json.Unmarshal(bytes, t) } -func (t TestGroup) GormDBDataType(db *gorm.DB, field *schema.Field) string { +func (t TestGroups) GormDBDataType(db *gorm.DB, field *schema.Field) string { switch db.Dialector.Name() { case "mysql", "sqlite": return "JSON" @@ -80,17 +82,29 @@ func (t TestGroup) GormDBDataType(db *gorm.DB, field *schema.Field) string { type Problem struct { gorm.Model - ProblemType ProblemType + ProblemType ProblemType `yaml:"ProblemType"` + + TimeLimit customfields.Time `yaml:"TimeLimit"` + MemoryLimit customfields.Memory `yaml:"MemoryLimit"` + + TestsNumber uint64 `yaml:"TestsNumber"` // TestGroups ignored for ICPC problems - TestGroups []TestGroup + TestGroups TestGroups `yaml:"TestGroups"` + + // WallTimeLimit specifies maximum execution and wait time. + // By default, it is max(5s, TimeLimit * 2) + WallTimeLimit *customfields.Time `yaml:"WallTimeLimit,omitempty"` - TimeLimit customfields.Time - MemoryLimit customfields.Memory - WallTimeLimit *customfields.Time + // MaxOpenFiles specifies the maximum number of files, opened by testing system. + // By default, it is 64 + MaxOpenFiles *uint64 `yaml:"MaxOpenFiles,omitempty"` - TestsNumber uint64 + // MaxThreads specifies the maximum number of threads and/or processes; + // By default, it is a single thread + // If MaxThreads equals to -1, any number of threads allowed + MaxThreads *int64 `yaml:"MaxThreads,omitempty"` - MaxOpenFiles *uint64 - MaxThreads *uint64 - MaxOutputSize *customfields.Memory + // MaxOutputSize specifies maximum output in EACH file. + // By default, it is 1g + MaxOutputSize *customfields.Memory `yaml:"MaxOutputSize,omitempty"` } diff --git a/common/db/models/submission.go b/common/db/models/submission.go index 1a24569..2b63c17 100644 --- a/common/db/models/submission.go +++ b/common/db/models/submission.go @@ -79,8 +79,8 @@ type Submission struct { ProblemID uint64 `json:"ProblemID" yaml:"ProblemID"` Language string `json:"Language" yaml:"Language"` - Score float64 `json:"Score" yaml:"Score"` - Verdict verdict.Verdict `json:"Verdict" yaml:"Verdict"` - TestResults TestResults `gorm:"type:jsonb" json:"TestResults" yaml:"TestResults"` + Score float64 `json:"GroupScore" yaml:"GroupScore"` + Verdict verdict.Verdict `json:"Verdict" yaml:"Verdict"` + TestResults TestResults `gorm:"type:jsonb" json:"TestResults" yaml:"TestResults"` GroupResults GroupResults } diff --git a/go.mod b/go.mod index 9e5e4c3..e8e6404 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/stretchr/testify v1.10.0 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 - github.com/swaggo/swag v1.16.4 github.com/xorcare/pointer v1.2.2 golang.org/x/net v0.39.0 gopkg.in/yaml.v3 v3.0.1 @@ -53,6 +52,7 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/swaggo/swag v1.16.4 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.16.0 // indirect From 132ecabf8ae6f8c5d4793639926354bf76281ba4 Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Sat, 3 May 2025 12:03:29 +0800 Subject: [PATCH 08/10] fixes --- common/db/models/problem.go | 14 +- common/db/models/submission.go | 6 +- master/queue/jobgenerators/generator_test.go | 90 ++++++ master/queue/jobgenerators/icpc_generator.go | 2 +- master/queue/jobgenerators/ioi_generator.go | 314 ++++++++++--------- 5 files changed, 275 insertions(+), 151 deletions(-) diff --git a/common/db/models/problem.go b/common/db/models/problem.go index a6b56a8..2d1d378 100644 --- a/common/db/models/problem.go +++ b/common/db/models/problem.go @@ -11,16 +11,14 @@ import ( type ProblemType int -// TestGroupScoringType sets how should scheduler set points for a group -type TestGroupScoringType int - -// TestGroupFeedbackType sets which info about tests in a group would be shown -type TestGroupFeedbackType int - const ( ProblemTypeICPC ProblemType = iota + 1 ProblemTypeIOI ) + +// TestGroupScoringType sets how should scheduler set points for a group +type TestGroupScoringType int + const ( // TestGroupScoringTypeComplete means that group costs TestGroup.GroupScore (all the tests should be OK) TestGroupScoringTypeComplete TestGroupScoringType = iota + 1 @@ -29,6 +27,10 @@ const ( // TestGroupScoringTypeMin means that group score = min(checker's scores among all the tests) TestGroupScoringTypeMin ) + +// TestGroupFeedbackType sets which info about tests in a group would be shown +type TestGroupFeedbackType int + const ( // TestGroupFeedbackTypeNone won't show anything TestGroupFeedbackTypeNone TestGroupFeedbackType = iota + 1 diff --git a/common/db/models/submission.go b/common/db/models/submission.go index 2b63c17..cce1a34 100644 --- a/common/db/models/submission.go +++ b/common/db/models/submission.go @@ -79,8 +79,8 @@ type Submission struct { ProblemID uint64 `json:"ProblemID" yaml:"ProblemID"` Language string `json:"Language" yaml:"Language"` - Score float64 `json:"GroupScore" yaml:"GroupScore"` + Score float64 `json:"Score" yaml:"Score"` Verdict verdict.Verdict `json:"Verdict" yaml:"Verdict"` - TestResults TestResults `gorm:"type:jsonb" json:"TestResults" yaml:"TestResults"` - GroupResults GroupResults + TestResults TestResults `json:"TestResults" yaml:"TestResults"` + GroupResults GroupResults `json:"GroupResults" yaml:"GroupResults"` } diff --git a/master/queue/jobgenerators/generator_test.go b/master/queue/jobgenerators/generator_test.go index 70d9d76..b043e9b 100644 --- a/master/queue/jobgenerators/generator_test.go +++ b/master/queue/jobgenerators/generator_test.go @@ -711,6 +711,96 @@ func TestIOIGenerator(t *testing.T) { }, }) }) + + t.Run("OK, run, FAIL, get", func(t *testing.T) { + baseStat := &masterconn.JobResultStatistics{ + Time: 100, + Memory: 100, + WallTime: 100, + ExitCode: 0, + } + problem := models.Problem{ + ProblemType: models.ProblemTypeIOI, + TestsNumber: 4, + TestGroups: []models.TestGroup{ + { + Name: "group1", + FirstTest: 1, + LastTest: 4, + TestScore: nil, + GroupScore: pointer.Float64(100), + ScoringType: models.TestGroupScoringTypeComplete, + RequiredGroupNames: make([]string, 0), + }, + }, + } + gen, err := NewGenerator(&problem, fixtureSubmission(1)) + require.NoError(t, err) + job := nextJob(t, gen, 1, invokerconn.CompileJob, 0) + sub, err := gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job.ID, + Verdict: verdict.CD, + }) + require.Nil(t, sub) + require.NoError(t, err) + job1 := nextJob(t, gen, 1, invokerconn.TestJob, 1) + job2 := nextJob(t, gen, 1, invokerconn.TestJob, 2) + job3 := nextJob(t, gen, 1, invokerconn.TestJob, 3) + // now finish 1 and 3 + sub, err = gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job1.ID, + Verdict: verdict.OK, + }) + require.NoError(t, err) + require.Nil(t, sub) + sub, err = gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job3.ID, + Verdict: verdict.WA, + Statistics: baseStat, + }) + // this group is already failed, so the generator should not return any job + require.Nil(t, gen.NextJob()) + sub, err = gen.JobCompleted(&masterconn.InvokerJobResult{ + JobID: job2.ID, + Verdict: verdict.TL, + Statistics: baseStat, + }) + require.NoError(t, err) + require.Nil(t, gen.NextJob()) + require.NotNil(t, sub) + + require.Equal(t, verdict.PT, sub.Verdict) + require.Equal(t, 0., sub.Score) + require.Equal(t, sub.GroupResults, models.GroupResults{ + { + GroupName: "group1", + Points: 0., + Passed: false, + }, + }) + require.Equal(t, models.TestResult{ + TestNumber: 1, + Points: nil, + Verdict: verdict.OK, + }, sub.TestResults[0]) + require.Equal(t, models.TestResult{ + TestNumber: 2, + Points: nil, + Verdict: verdict.TL, + Time: 100, + Memory: 100, + }, sub.TestResults[1]) + require.Equal(t, models.TestResult{ + TestNumber: 3, + Points: nil, + Verdict: verdict.SK, + }, sub.TestResults[2]) + require.Equal(t, models.TestResult{ + TestNumber: 4, + Points: nil, + Verdict: verdict.SK, + }, sub.TestResults[3]) + }) }) t.Run("Fails in TestGroupScoringTypeEachTest", func(t *testing.T) { diff --git a/master/queue/jobgenerators/icpc_generator.go b/master/queue/jobgenerators/icpc_generator.go index 462df50..005d89c 100644 --- a/master/queue/jobgenerators/icpc_generator.go +++ b/master/queue/jobgenerators/icpc_generator.go @@ -122,7 +122,7 @@ func (i *ICPCGenerator) JobCompleted(result *masterconn.InvokerJobResult) (*mode defer i.mutex.Unlock() job, ok := i.givenJobs[result.JobID] if !ok { - return nil, fmt.Errorf("job %s not exist", result.JobID) + return nil, fmt.Errorf("job %s does not exist", result.JobID) } delete(i.givenJobs, result.JobID) diff --git a/master/queue/jobgenerators/ioi_generator.go b/master/queue/jobgenerators/ioi_generator.go index 888a93b..4441bf4 100644 --- a/master/queue/jobgenerators/ioi_generator.go +++ b/master/queue/jobgenerators/ioi_generator.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/google/uuid" "github.com/xorcare/pointer" - "math" "slices" "sync" "testing_system/common/connectors/invokerconn" @@ -26,20 +25,14 @@ type IOIGenerator struct { testNumberToGroupName map[uint64]string internalTestResults []*internalTestResult - // firstNotCompletedTest = the longest prefix of the tests, for which we know verdict + // firstNotCompletedTest = the longest prefix of the tests, for which we know verdict; 1-based indexing firstNotCompletedTest uint64 - // firstNotCompletedGroup = the longest prefix of the groups, for which we know verdict + // firstNotCompletedGroup = the longest prefix of the groups, for which we know verdict; 1-based indexing firstNotCompletedGroup uint64 - firstUnseenTest uint64 + // firstNotGivenTest = first test with internalTestState = testNotGiven; 1-based indexing + firstNotGivenTest uint64 } -type internalGroupState int - -const ( - groupRunning internalGroupState = iota - groupFailed // if the group isn't failed and completed, it will still have groupRunning state -) - type internalTestState int const ( @@ -49,8 +42,8 @@ const ( ) type internalGroupInfo struct { - state internalGroupState - shouldSkipTests bool + shouldGiveNewJobs bool + shouldMarkFinalTestsSkipped bool } type internalTestResult struct { @@ -125,8 +118,8 @@ func (i *IOIGenerator) prepareGenerator() error { } i.groupNameToGroupInfo[group.Name] = group i.groupNameToInternalInfo[group.Name] = &internalGroupInfo{ - state: groupRunning, - shouldSkipTests: false, + shouldGiveNewJobs: true, + shouldMarkFinalTestsSkipped: false, } } return i.checkIfGroupsDependenciesOK(problem) @@ -139,7 +132,7 @@ func (i *IOIGenerator) ID() string { func (i *IOIGenerator) NextJob() *invokerconn.Job { i.mutex.Lock() defer i.mutex.Unlock() - if i.state == compilationFinished && i.firstUnseenTest > i.problem.TestsNumber { + if i.state == compilationFinished && i.firstNotGivenTest > i.problem.TestsNumber { return nil } if i.state == compilationStarted { @@ -160,113 +153,133 @@ func (i *IOIGenerator) NextJob() *invokerconn.Job { return job } job.Type = invokerconn.TestJob - for i.firstUnseenTest < i.problem.TestsNumber { - groupName := i.testNumberToGroupName[i.firstUnseenTest] + for i.firstNotGivenTest <= i.problem.TestsNumber { + groupName := i.testNumberToGroupName[i.firstNotGivenTest-1] groupInfo := i.groupNameToInternalInfo[groupName] - if groupInfo.shouldSkipTests { - i.firstUnseenTest++ + if !groupInfo.shouldGiveNewJobs { + i.internalTestResults[i.firstNotGivenTest-1].state = testFinished + i.firstNotGivenTest++ continue } - i.internalTestResults[i.firstUnseenTest].state = testRunning - i.firstUnseenTest++ - job.Test = i.firstUnseenTest + i.internalTestResults[i.firstNotGivenTest-1].state = testRunning + job.Test = i.firstNotGivenTest i.givenJobs[job.ID] = job + i.firstNotGivenTest++ return job } return nil } -func isTestFailed(testGroupInfo models.TestGroup, testVerdict verdict.Verdict) bool { +func doesTestPreventTestingGroup(testGroupInfo models.TestGroup, testVerdict verdict.Verdict) bool { switch testGroupInfo.ScoringType { case models.TestGroupScoringTypeComplete: return testVerdict != verdict.OK case models.TestGroupScoringTypeEachTest: - return testVerdict != verdict.OK + return false case models.TestGroupScoringTypeMin: - return testVerdict != verdict.OK + return testVerdict != verdict.OK && testVerdict != verdict.PT default: - panic(fmt.Sprintf("unknown testGroupInfo.ScoringType %v", testGroupInfo.ScoringType)) + logger.Panic("unknown testGroupInfo.ScoringType %v", testGroupInfo.ScoringType) } + return false } -func shouldSkipNewTests(testGroupInfo models.TestGroup, testVerdict verdict.Verdict) bool { - switch testGroupInfo.ScoringType { +func (i *IOIGenerator) calcGroupVerdict( + groupInfo models.TestGroup) verdict.Verdict { + firstTest, lastTest := groupInfo.FirstTest, groupInfo.LastTest + if slices.IndexFunc(i.submission.TestResults[firstTest-1:lastTest], func(result models.TestResult) bool { + return result.Verdict != verdict.OK + }) != -1 { + return verdict.PT + } else { + return verdict.OK + } +} + +func (i *IOIGenerator) completeGroupTesting(groupInfo models.TestGroup) { + if i.firstNotCompletedTest != groupInfo.LastTest { + logger.Panic("completeGroupTesting called, but wasn't finished right now") + } + score := 0.0 + groupVerdict := i.calcGroupVerdict(groupInfo) + switch groupInfo.ScoringType { case models.TestGroupScoringTypeComplete: - return testVerdict != verdict.OK + if groupVerdict == verdict.OK { + score = *groupInfo.GroupScore + } case models.TestGroupScoringTypeEachTest: - return false + for testNumber := groupInfo.FirstTest; testNumber <= groupInfo.LastTest; testNumber++ { + testScore := i.submission.TestResults[testNumber-1].Points + if testScore == nil { + logger.Panic("test %v has points in group %v", testNumber, groupInfo.Name) + } else { + score += *testScore + } + } case models.TestGroupScoringTypeMin: - return testVerdict != verdict.OK && testVerdict != verdict.PT - default: - panic(fmt.Sprintf("unknown testGroupInfo.ScoringType %v", testGroupInfo.ScoringType)) + score = *groupInfo.GroupScore + for testNumber := groupInfo.FirstTest; testNumber <= groupInfo.LastTest; testNumber++ { + testScore := i.submission.TestResults[testNumber-1].Points + if testScore == nil { + if i.submission.TestResults[testNumber-1].Verdict != verdict.SK { + logger.Panic("test %v has points in group %v", testNumber, groupInfo.Name) + } else { + score = 0 + } + } else { + score = min(score, *testScore) + } + } } + i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ + GroupName: groupInfo.Name, + Points: score, + Passed: groupVerdict == verdict.OK, + }) + i.firstNotCompletedGroup++ } // increaseCompletedTestsAndGroups must be done with acquired mutex func (i *IOIGenerator) increaseCompletedTestsAndGroups() { TestsLoop: - for i.firstNotCompletedTest != i.problem.TestsNumber { - testGroupName := i.testNumberToGroupName[i.firstNotCompletedTest] + for i.firstNotCompletedTest <= i.problem.TestsNumber { + testGroupName := i.testNumberToGroupName[i.firstNotCompletedTest-1] testGroupInfo := i.groupNameToGroupInfo[testGroupName] testInternalGroupInfo := i.groupNameToInternalInfo[testGroupName] - switch i.internalTestResults[i.firstNotCompletedTest].state { + switch i.internalTestResults[i.firstNotCompletedTest-1].state { case testNotGiven: - if !testInternalGroupInfo.shouldSkipTests { + if !testInternalGroupInfo.shouldMarkFinalTestsSkipped { break TestsLoop } case testRunning: break TestsLoop case testFinished: } - i.submission.TestResults = append(i.submission.TestResults, - *i.internalTestResults[i.firstNotCompletedTest].result) - if !testInternalGroupInfo.shouldSkipTests { - if shouldSkipNewTests(testGroupInfo, i.submission.TestResults[i.firstNotCompletedTest].Verdict) { - testInternalGroupInfo.shouldSkipTests = true - // update shouldSkipTests + + if testInternalGroupInfo.shouldMarkFinalTestsSkipped { + i.submission.TestResults = append(i.submission.TestResults, + models.TestResult{ + TestNumber: i.internalTestResults[i.firstNotCompletedTest-1].result.TestNumber, + Verdict: verdict.SK, + }) + } else { + i.submission.TestResults = append(i.submission.TestResults, + *i.internalTestResults[i.firstNotCompletedTest-1].result) + testVerdict := i.internalTestResults[i.firstNotCompletedTest-1].result.Verdict + if doesTestPreventTestingGroup(testGroupInfo, testVerdict) { + testInternalGroupInfo.shouldMarkFinalTestsSkipped = true for _, group := range i.problem.TestGroups { for _, requiredGroupName := range group.RequiredGroupNames { - if i.groupNameToInternalInfo[requiredGroupName].shouldSkipTests { - i.groupNameToInternalInfo[group.Name].shouldSkipTests = true + if i.groupNameToInternalInfo[requiredGroupName].shouldMarkFinalTestsSkipped { + i.groupNameToInternalInfo[group.Name].shouldMarkFinalTestsSkipped = true } } } } - } else { - i.submission.TestResults[i.firstNotCompletedTest].Verdict = verdict.SK } - if i.firstNotCompletedTest+1 == testGroupInfo.LastTest { - score := 0.0 - switch testGroupInfo.ScoringType { - case models.TestGroupScoringTypeComplete: - if testInternalGroupInfo.state != groupFailed { - score = *testGroupInfo.GroupScore - } - case models.TestGroupScoringTypeEachTest: - for testNumber := testGroupInfo.FirstTest - 1; testNumber < testGroupInfo.LastTest; testNumber++ { - if testScore := i.submission.TestResults[testNumber].Points; testScore != nil { - score += *testScore - } - } - case models.TestGroupScoringTypeMin: - score = math.Inf(+1) - for testNumber := testGroupInfo.FirstTest - 1; testNumber < testGroupInfo.LastTest; testNumber++ { - if testScore := i.submission.TestResults[testNumber].Points; testScore != nil { - score = min(score, *testScore) - } else { - score = 0.0 - } - } - if score == math.Inf(+1) { - score = 0 - } - } - i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ - GroupName: testGroupName, - Points: score, - Passed: testInternalGroupInfo.state != groupFailed, - }) - i.firstNotCompletedGroup++ + + if i.firstNotCompletedTest == testGroupInfo.LastTest { + i.completeGroupTesting(testGroupInfo) } i.firstNotCompletedTest++ } @@ -283,18 +296,20 @@ func (i *IOIGenerator) compileJobCompleted(job *invokerconn.Job, result *masterc return nil, nil case verdict.CE: i.submission.Verdict = verdict.CE - for _, group := range i.problem.TestGroups { - i.submission.GroupResults = append(i.submission.GroupResults, models.GroupResult{ - GroupName: group.Name, - Points: 0, - Passed: false, - }) - } for t := range i.problem.TestsNumber { - i.submission.TestResults = append(i.submission.TestResults, models.TestResult{ - TestNumber: t + 1, - Verdict: verdict.SK, - }) + i.internalTestResults[t] = &internalTestResult{ + result: &models.TestResult{ + TestNumber: t + 1, + Points: nil, + Verdict: verdict.SK, + }, + state: testFinished, + } + } + i.increaseCompletedTestsAndGroups() + if i.firstNotCompletedTest <= i.problem.TestsNumber || + int(i.firstNotCompletedGroup) <= len(i.problem.TestGroups) { + logger.Panic("not all tests got verdict after CE") } return i.submission, nil default: @@ -327,18 +342,70 @@ func populateTestJobResult(groupInfo models.TestGroup, result *masterconn.Invoke } } default: - panic(fmt.Sprintf("unknown group scoring type: %v", groupInfo.ScoringType)) + logger.Panic("unknown group scoring type: %v", groupInfo.ScoringType) } return nil } +func (i *IOIGenerator) stopGivingNewTestsIfNeeded( + testInternalGroupInfo *internalGroupInfo, testGroupInfo models.TestGroup, testVerdict verdict.Verdict) { + if !doesTestPreventTestingGroup(testGroupInfo, testVerdict) { + return + } + + testInternalGroupInfo.shouldGiveNewJobs = false + for _, group := range i.problem.TestGroups { + for _, requiredGroupName := range group.RequiredGroupNames { + if !i.groupNameToInternalInfo[requiredGroupName].shouldGiveNewJobs { + i.groupNameToInternalInfo[group.Name].shouldGiveNewJobs = false + break + } + } + } +} + +func (i *IOIGenerator) setFinalScoreAndVerdict() { + if len(i.givenJobs) != 0 { + logger.Panic("setFinalScoreAndVerdict called, but there are some jobs still") + } + if i.firstNotCompletedTest != i.problem.TestsNumber+1 { + logger.Panic("setFinalScoreAndVerdict called, but not all the tests were completed") + } + if len(i.submission.GroupResults) != len(i.problem.TestGroups) { + logger.Panic("for some reason \"len(i.submission.GroupResults) != len(i.problem.TestGroups)\"") + } + if len(i.submission.TestResults) != int(i.problem.TestsNumber) { + logger.Panic("for some reason \"len(i.submission.TestResults) != int(i.problem.TestsNumber)\"") + } + for _, groupResult := range i.submission.GroupResults { + i.submission.Score += groupResult.Points + } + + hasSkipped := false + hasVerdict := false + for _, testResult := range i.submission.TestResults { + if testResult.Verdict != verdict.OK && testResult.Verdict != verdict.SK { + i.submission.Verdict = verdict.PT + hasVerdict = true + } else if testResult.Verdict == verdict.SK { + hasSkipped = true + } + } + if !hasVerdict { + if hasSkipped { + logger.Panic("problem does not have verdict, but have skipped tests") + } + i.submission.Verdict = verdict.OK + } + return +} + // testJobCompleted must be done with acquired mutex func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn.InvokerJobResult) (*models.Submission, error) { if job.Type != invokerconn.TestJob { return nil, fmt.Errorf("job type %v is not test job", job.ID) } - test := job.Test - 1 - testGroupName := i.testNumberToGroupName[test] + testGroupName := i.testNumberToGroupName[job.Test-1] testGroupInfo := i.groupNameToGroupInfo[testGroupName] testInternalGroupInfo := i.groupNameToInternalInfo[testGroupName] // move info from result to internal state @@ -352,54 +419,19 @@ func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn } result.Error += err.Error() } - i.internalTestResults[test].result.Points = result.Points - i.internalTestResults[test].result.Verdict = result.Verdict - i.internalTestResults[test].result.Error = result.Error + i.internalTestResults[job.Test-1].result.Points = result.Points + i.internalTestResults[job.Test-1].result.Verdict = result.Verdict + i.internalTestResults[job.Test-1].result.Error = result.Error if result.Statistics != nil { - i.internalTestResults[test].result.Time = result.Statistics.Time - i.internalTestResults[test].result.Memory = result.Statistics.Memory + i.internalTestResults[job.Test-1].result.Time = result.Statistics.Time + i.internalTestResults[job.Test-1].result.Memory = result.Statistics.Memory } - i.internalTestResults[test].state = testFinished + i.internalTestResults[job.Test-1].state = testFinished - if isTestFailed(testGroupInfo, result.Verdict) { - testInternalGroupInfo.state = groupFailed - for _, group := range i.problem.TestGroups { - for _, requiredGroupName := range group.RequiredGroupNames { - if i.groupNameToInternalInfo[requiredGroupName].state == groupFailed { - i.groupNameToInternalInfo[group.Name].state = groupFailed - break - } - } - } - } i.increaseCompletedTestsAndGroups() - if i.firstNotCompletedTest == i.problem.TestsNumber && len(i.givenJobs) == 0 { - if len(i.submission.GroupResults) != len(i.problem.TestGroups) { - panic("for some reason \"len(i.submission.GroupResults) != len(i.problem.TestGroups)\"") - } - if len(i.submission.TestResults) != int(i.problem.TestsNumber) { - panic("for some reason \"len(i.submission.TestResults) != int(i.problem.TestsNumber)\"") - } - for _, groupResult := range i.submission.GroupResults { - i.submission.Score += groupResult.Points - } - - haveSkipped := false - haveVerdict := false - for _, testResult := range i.submission.TestResults { - if testResult.Verdict != verdict.OK && testResult.Verdict != verdict.SK { - i.submission.Verdict = verdict.PT - haveVerdict = true - } else if testResult.Verdict == verdict.SK { - haveSkipped = true - } - } - if !haveVerdict { - if haveSkipped { - panic("problem does not have verdict, but have skipped tests") - } - i.submission.Verdict = verdict.OK - } + i.stopGivingNewTestsIfNeeded(testInternalGroupInfo, testGroupInfo, result.Verdict) + if i.firstNotCompletedTest > i.problem.TestsNumber && len(i.givenJobs) == 0 { + i.setFinalScoreAndVerdict() return i.submission, nil } return nil, nil @@ -438,9 +470,9 @@ func NewIOIGenerator(problem *models.Problem, submission *models.Submission) (Ge groupNameToInternalInfo: make(map[string]*internalGroupInfo), testNumberToGroupName: make(map[uint64]string), internalTestResults: make([]*internalTestResult, 0), - firstNotCompletedTest: 0, - firstNotCompletedGroup: 0, - firstUnseenTest: 0, + firstNotCompletedTest: 1, + firstNotCompletedGroup: 1, + firstNotGivenTest: 1, } if err = generator.prepareGenerator(); err != nil { return nil, err From ca052134774f009571680d7e2f33614a02498008 Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Sat, 3 May 2025 12:14:19 +0800 Subject: [PATCH 09/10] more checks at problem creation --- master/queue/jobgenerators/generator_test.go | 59 ++++++++++++++++++-- master/queue/jobgenerators/ioi_generator.go | 3 + 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/master/queue/jobgenerators/generator_test.go b/master/queue/jobgenerators/generator_test.go index b043e9b..7646a41 100644 --- a/master/queue/jobgenerators/generator_test.go +++ b/master/queue/jobgenerators/generator_test.go @@ -290,6 +290,21 @@ func TestIOIGenerator(t *testing.T) { }, TestsNumber: 1, }, + // type Min, but GroupScore is nil + { + ProblemType: models.ProblemTypeIOI, + TestGroups: []models.TestGroup{ + { + Name: "name", + FirstTest: 1, + LastTest: 1, + GroupScore: nil, + ScoringType: models.TestGroupScoringTypeMin, + RequiredGroupNames: make([]string, 0), + }, + }, + TestsNumber: 1, + }, // cyclic groups { ProblemType: models.ProblemTypeIOI, @@ -298,6 +313,7 @@ func TestIOIGenerator(t *testing.T) { Name: "name1", FirstTest: 1, LastTest: 1, + GroupScore: pointer.Float64(1.0), ScoringType: models.TestGroupScoringTypeMin, RequiredGroupNames: []string{"name2"}, }, @@ -305,12 +321,13 @@ func TestIOIGenerator(t *testing.T) { Name: "name2", FirstTest: 2, LastTest: 2, + GroupScore: pointer.Float64(1.0), ScoringType: models.TestGroupScoringTypeMin, RequiredGroupNames: []string{"name1"}, }, }, }, - // test not covered + // test is not covered { ProblemType: models.ProblemTypeIOI, TestGroups: []models.TestGroup{ @@ -318,7 +335,7 @@ func TestIOIGenerator(t *testing.T) { Name: "name", FirstTest: 1, LastTest: 1, - TestScore: nil, + TestScore: pointer.Float64(1.0), ScoringType: models.TestGroupScoringTypeEachTest, RequiredGroupNames: make([]string, 0), }, @@ -333,7 +350,7 @@ func TestIOIGenerator(t *testing.T) { Name: "name", FirstTest: 1, LastTest: 2, - TestScore: nil, + TestScore: pointer.Float64(1.0), ScoringType: models.TestGroupScoringTypeEachTest, RequiredGroupNames: make([]string, 0), }, @@ -341,7 +358,7 @@ func TestIOIGenerator(t *testing.T) { Name: "name1", FirstTest: 2, LastTest: 3, - TestScore: nil, + TestScore: pointer.Float64(1.0), ScoringType: models.TestGroupScoringTypeEachTest, RequiredGroupNames: make([]string, 0), }, @@ -356,7 +373,7 @@ func TestIOIGenerator(t *testing.T) { Name: "name", FirstTest: 1, LastTest: 2, - TestScore: nil, + TestScore: pointer.Float64(1.0), ScoringType: models.TestGroupScoringTypeEachTest, RequiredGroupNames: make([]string, 0), }, @@ -364,13 +381,43 @@ func TestIOIGenerator(t *testing.T) { Name: "name", FirstTest: 3, LastTest: 3, - TestScore: nil, + TestScore: pointer.Float64(1.0), ScoringType: models.TestGroupScoringTypeEachTest, RequiredGroupNames: make([]string, 0), }, }, TestsNumber: 3, }, + // first test > last test + { + ProblemType: models.ProblemTypeIOI, + TestGroups: []models.TestGroup{ + { + Name: "name1", + FirstTest: 1, + LastTest: 2, + TestScore: pointer.Float64(1.0), + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + { + Name: "name2", + FirstTest: 2, + LastTest: 1, + TestScore: pointer.Float64(1.0), + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + { + Name: "name3", + FirstTest: 3, + LastTest: 3, + TestScore: pointer.Float64(1.0), + ScoringType: models.TestGroupScoringTypeEachTest, + RequiredGroupNames: make([]string, 0), + }, + }, + }, } for _, problem := range badProblems { _, err := NewIOIGenerator(&problem, &models.Submission{}) diff --git a/master/queue/jobgenerators/ioi_generator.go b/master/queue/jobgenerators/ioi_generator.go index 4441bf4..b2ce7f2 100644 --- a/master/queue/jobgenerators/ioi_generator.go +++ b/master/queue/jobgenerators/ioi_generator.go @@ -106,6 +106,9 @@ func (i *IOIGenerator) prepareGenerator() error { default: return fmt.Errorf("unknown TestGroupScoringType %v", group.ScoringType) } + if group.FirstTest > group.LastTest { + return fmt.Errorf("group %v has FirstTest > LastTest", group.Name) + } for testNumber := group.FirstTest; testNumber <= group.LastTest; testNumber++ { i.testNumberToGroupName[testNumber-1] = group.Name i.internalTestResults = append(i.internalTestResults, &internalTestResult{ From 263d7bf6bf058f9c9b7b7d44eac3b5f1b7e7c175 Mon Sep 17 00:00:00 2001 From: Mikhail Kondrashin Date: Tue, 6 May 2025 01:49:53 +0900 Subject: [PATCH 10/10] fixes v2 --- master/queue/jobgenerators/ioi_generator.go | 73 +++++++++++---------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/master/queue/jobgenerators/ioi_generator.go b/master/queue/jobgenerators/ioi_generator.go index b2ce7f2..f4049fb 100644 --- a/master/queue/jobgenerators/ioi_generator.go +++ b/master/queue/jobgenerators/ioi_generator.go @@ -187,8 +187,7 @@ func doesTestPreventTestingGroup(testGroupInfo models.TestGroup, testVerdict ver return false } -func (i *IOIGenerator) calcGroupVerdict( - groupInfo models.TestGroup) verdict.Verdict { +func (i *IOIGenerator) calcGroupVerdict(groupInfo models.TestGroup) verdict.Verdict { firstTest, lastTest := groupInfo.FirstTest, groupInfo.LastTest if slices.IndexFunc(i.submission.TestResults[firstTest-1:lastTest], func(result models.TestResult) bool { return result.Verdict != verdict.OK @@ -242,8 +241,8 @@ func (i *IOIGenerator) completeGroupTesting(groupInfo models.TestGroup) { i.firstNotCompletedGroup++ } -// increaseCompletedTestsAndGroups must be done with acquired mutex -func (i *IOIGenerator) increaseCompletedTestsAndGroups() { +// updateSubmissionResult must be done with acquired mutex +func (i *IOIGenerator) updateSubmissionResult() (*models.Submission, error) { TestsLoop: for i.firstNotCompletedTest <= i.problem.TestsNumber { testGroupName := i.testNumberToGroupName[i.firstNotCompletedTest-1] @@ -257,6 +256,7 @@ TestsLoop: case testRunning: break TestsLoop case testFinished: + break } if testInternalGroupInfo.shouldMarkFinalTestsSkipped { @@ -286,43 +286,41 @@ TestsLoop: } i.firstNotCompletedTest++ } + if i.firstNotCompletedTest > i.problem.TestsNumber && len(i.givenJobs) == 0 { + i.setFinalScoreAndVerdict() + if int(i.firstNotCompletedGroup) <= len(i.problem.TestGroups) { + logger.Panic("not all the groups were filled, but the problem is considered tested") + } + return i.submission, nil + } + return nil, nil } // compileJobCompleted must be done with acquired mutex -func (i *IOIGenerator) compileJobCompleted(job *invokerconn.Job, result *masterconn.InvokerJobResult) (*models.Submission, error) { +func (i *IOIGenerator) compileJobCompleted( + job *invokerconn.Job, + result *masterconn.InvokerJobResult, +) { if job.Type != invokerconn.CompileJob { - return nil, fmt.Errorf("job type %s is not compile job", job.ID) + logger.Warn("job type %s is %v; treating is as a compile job", job.ID, job.Type) } switch result.Verdict { case verdict.CD: i.state = compilationFinished - return nil, nil case verdict.CE: i.submission.Verdict = verdict.CE - for t := range i.problem.TestsNumber { - i.internalTestResults[t] = &internalTestResult{ - result: &models.TestResult{ - TestNumber: t + 1, - Points: nil, - Verdict: verdict.SK, - }, - state: testFinished, - } + for _, group := range i.problem.TestGroups { + i.groupNameToInternalInfo[group.Name].shouldMarkFinalTestsSkipped = true } - i.increaseCompletedTestsAndGroups() - if i.firstNotCompletedTest <= i.problem.TestsNumber || - int(i.firstNotCompletedGroup) <= len(i.problem.TestGroups) { - logger.Panic("not all tests got verdict after CE") - } - return i.submission, nil default: - return nil, fmt.Errorf("unknown verdict for compilation completed: %v", result.Verdict) + logger.Panic("unknown verdict for compilation completed: %v", result.Verdict) } } func populateTestJobResult(groupInfo models.TestGroup, result *masterconn.InvokerJobResult) error { switch groupInfo.ScoringType { case models.TestGroupScoringTypeComplete: + break case models.TestGroupScoringTypeEachTest: if result.Points == nil { if result.Verdict == verdict.OK { @@ -351,7 +349,10 @@ func populateTestJobResult(groupInfo models.TestGroup, result *masterconn.Invoke } func (i *IOIGenerator) stopGivingNewTestsIfNeeded( - testInternalGroupInfo *internalGroupInfo, testGroupInfo models.TestGroup, testVerdict verdict.Verdict) { + testInternalGroupInfo *internalGroupInfo, + testGroupInfo models.TestGroup, + testVerdict verdict.Verdict, +) { if !doesTestPreventTestingGroup(testGroupInfo, testVerdict) { return } @@ -383,6 +384,10 @@ func (i *IOIGenerator) setFinalScoreAndVerdict() { for _, groupResult := range i.submission.GroupResults { i.submission.Score += groupResult.Points } + if i.submission.Verdict != verdict.RU { + logger.Trace("submission %v already has verdict %v", i.submission, i.submission.Verdict) + return + } hasSkipped := false hasVerdict := false @@ -404,9 +409,12 @@ func (i *IOIGenerator) setFinalScoreAndVerdict() { } // testJobCompleted must be done with acquired mutex -func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn.InvokerJobResult) (*models.Submission, error) { +func (i *IOIGenerator) testJobCompleted( + job *invokerconn.Job, + result *masterconn.InvokerJobResult, +) { if job.Type != invokerconn.TestJob { - return nil, fmt.Errorf("job type %v is not test job", job.ID) + logger.Warn("job type %s is %v; treating is as a testing job", job.ID, job.Type) } testGroupName := i.testNumberToGroupName[job.Test-1] testGroupInfo := i.groupNameToGroupInfo[testGroupName] @@ -430,14 +438,7 @@ func (i *IOIGenerator) testJobCompleted(job *invokerconn.Job, result *masterconn i.internalTestResults[job.Test-1].result.Memory = result.Statistics.Memory } i.internalTestResults[job.Test-1].state = testFinished - - i.increaseCompletedTestsAndGroups() i.stopGivingNewTestsIfNeeded(testInternalGroupInfo, testGroupInfo, result.Verdict) - if i.firstNotCompletedTest > i.problem.TestsNumber && len(i.givenJobs) == 0 { - i.setFinalScoreAndVerdict() - return i.submission, nil - } - return nil, nil } func (i *IOIGenerator) JobCompleted(jobResult *masterconn.InvokerJobResult) (*models.Submission, error) { @@ -450,12 +451,13 @@ func (i *IOIGenerator) JobCompleted(jobResult *masterconn.InvokerJobResult) (*mo delete(i.givenJobs, jobResult.JobID) switch job.Type { case invokerconn.CompileJob: - return i.compileJobCompleted(job, jobResult) + i.compileJobCompleted(job, jobResult) case invokerconn.TestJob: - return i.testJobCompleted(job, jobResult) + i.testJobCompleted(job, jobResult) default: return nil, fmt.Errorf("unknown job type for IOI problem: %v", job.Type) } + return i.updateSubmissionResult() } func NewIOIGenerator(problem *models.Problem, submission *models.Submission) (Generator, error) { @@ -477,6 +479,7 @@ func NewIOIGenerator(problem *models.Problem, submission *models.Submission) (Ge firstNotCompletedGroup: 1, firstNotGivenTest: 1, } + generator.submission.Verdict = verdict.RU if err = generator.prepareGenerator(); err != nil { return nil, err }