Merge branch 'main' into dev/storage

Author: Artem Titov

common/config/config.go

@@ -15,7 +15,7 @@ type Config struct {
 	LogLevel *int    `yaml:"LogLevel,omitempty"`
 
 	Invoker *InvokerConfig `yaml:"Invoker,omitempty"`
-	Storage *StorageConfig `yaml:"Invoker,omitempty"`
+	Storage *StorageConfig `yaml:"Storage,omitempty"`
 	// TODO: Add instances here
 
 	DB DBConfig `yaml:"DB"`
@@ -47,4 +47,10 @@ func fillInConfig(config *Config) {
 	}
 
 	fillInConnections(config)
+	if config.Invoker != nil {
+		fillInInvokerConfig(config.Invoker)
+	}
+	if config.Storage != nil {
+		FillInStorageConfig(config.Storage)
+	}
 }

common/config/connection.go

@@ -6,5 +6,5 @@ type Connection struct {
 }
 
 func fillInConnections(config *Config) {
-	// TODO: Add auto connection creation if master or storageconn are set up
+	// TODO: Add auto connection creation if master or storage are set up
 }

common/config/invoker.go

@@ -1,18 +1,28 @@
 package config
 
-import "github.com/xorcare/pointer"
-
 type InvokerConfig struct {
-	Instances int  `yaml:"Instances"`
-	QueueSize *int `yaml:"QueueSize,omitempty"` // default is 10
+	Threads   uint64 `yaml:"Threads"`
+	Sandboxes uint64 `yaml:"Sandboxes"`
+	QueueSize uint64 `yaml:"QueueSize"`
 
 	CacheSize uint64 `yaml:"CacheSize"`
 	CachePath string `yaml:"CachePath"`
+
+	SaveOutputHead *uint64 `yaml:"SaveOutputHead,omitempty"`
+
+	CompilerConfigsFolder string `yaml:"CompilerConfigsFolder"`
 }
 
 func fillInInvokerConfig(config *InvokerConfig) {
-	if config.QueueSize == nil {
-		config.QueueSize = pointer.Int(10)
+	if config.Threads == 0 {
+		config.Threads = 1
+	}
+	if config.Sandboxes == 0 {
+		// TODO: find out what number is the best
+		config.Sandboxes = 1
+	}
+	if config.QueueSize == 0 {
+		config.QueueSize = 10
 	}
 	if len(config.CachePath) == 0 {
 		panic("No invoker cache path specified")

common/connectors/connector_base.go

@@ -0,0 +1,26 @@
+package connectors
+
+import (
+	"github.com/go-resty/resty/v2"
+	"testing_system/common/config"
+)
+
+type ConnectorBase struct {
+	Connection *config.Connection
+	client     *resty.Client
+}
+
+func NewConnectorBase(connection *config.Connection) *ConnectorBase {
+	c := &ConnectorBase{
+		Connection: connection,
+		client:     resty.New(),
+	}
+	// TODO: Add host initialization from connection
+	// TODO: Add auth
+	// TODO: Add retry configuration
+	return c
+}
+
+func (c *ConnectorBase) R() *resty.Request {
+	return c.client.R()
+}

common/connectors/invokerconn/connector.go

@@ -0,0 +1,29 @@
+package invokerconn
+
+import (
+	"github.com/go-resty/resty/v2"
+	"testing_system/common/config"
+	"testing_system/common/connectors"
+	"testing_system/lib/connector"
+)
+
+type Connector struct {
+	connection *connectors.ConnectorBase
+}
+
+func NewConnector(connection *config.Connection) *Connector {
+	return &Connector{connectors.NewConnectorBase(connection)}
+}
+
+func (c *Connector) Status() (*StatusResponse, error) {
+	r := c.connection.R()
+
+	return connector.Receive[StatusResponse](r, "/invoker/status", resty.MethodGet)
+}
+
+func (c *Connector) NewJob(job *Job) (*StatusResponse, error) {
+	r := c.connection.R()
+	r.SetBody(job)
+
+	return connector.Receive[StatusResponse](r, "/invoker/job/new", resty.MethodPost)
+}

common/connectors/invokerconn/structs.go

@@ -1,21 +1,23 @@
 package invokerconn
 
-import (
-	"testing_system/common/db/models"
-)
-
-// This will be changed in later commits
+type JobType int
 
 const (
-	JobTypeCompile = iota
-	JobTypeTest
+	CompileJob JobType = iota
+	TestJob
 )
 
 type Job struct {
-	SubmitID uint `json:"submitID" binding:"required"`
-	JobType  int  `json:"jobType" binding:"required"`
-	Test     int  `json:"test"`
+	ID       string  `json:"ID"`
+	SubmitID uint    `json:"SubmitID" binding:"required"`
+	Type     JobType `json:"JobType" binding:"required"`
+	Test     uint64  `json:"Test"`
+
+	// TODO: Add job dependency
+}
 
-	Submit  *models.Submission `json:"submit,omitempty"`
-	Problem *models.Problem    `json:"problem,omitempty"`
+type StatusResponse struct {
+	// TODO: add some information about invoker state
+	MaxNewJobs   uint64   `json:"MaxNewJobs"`
+	ActiveJobIDs []string `json:"ActiveJobIDs"`
 }

common/connectors/masterconn/connector.go

@@ -0,0 +1,15 @@
+package masterconn
+
+import "testing_system/common/config"
+
+type Connector struct {
+	// TODO: Add master connection
+}
+
+func NewConnector(connection *config.Connection) *Connector {
+	return nil
+}
+
+func (c *Connector) InvokerJobResult(result *InvokerJobResult) error {
+	return nil
+}

common/connectors/masterconn/structs.go

@@ -0,0 +1,16 @@
+package masterconn
+
+import (
+	"testing_system/common/connectors/invokerconn"
+	"testing_system/common/constants/verdict"
+)
+
+type InvokerJobResult struct {
+	JobID   string          `json:"JobID"`
+	Verdict verdict.Verdict `json:"Verdict"`
+	Error   string          `json:"Error,omitempty"` // Is set only in case of check failed
+
+	InvokerStatus *invokerconn.StatusResponse `json:"InvokerStatus"`
+
+	// TODO: Add some statistics (e.g time, memory, etc)
+}

common/connectors/storageconn/connector.go

@@ -0,0 +1,136 @@
+package storageconn
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing_system/common/config"
+	"testing_system/common/connectors"
+	"testing_system/common/constants/resource"
+
+	"github.com/go-resty/resty/v2"
+)
+
+type Connector struct {
+	connection *connectors.ConnectorBase
+}
+
+func NewConnector(connection *config.Connection) *Connector {
+	if connection == nil {
+		return nil
+	}
+	return &Connector{connectors.NewConnectorBase(connection)}
+}
+
+func (s *Connector) Download(request *Request) *ResponseFiles {
+	if err := os.MkdirAll(request.BaseFolder, 0775); err != nil {
+		return &ResponseFiles{Response: Response{R: *request, Error: fmt.Errorf("failed to create base folder: %v", err)}}
+	}
+
+	path := fmt.Sprintf("/storage/get?id=%d&dataType=%s&filepath=%s",
+		getIDForResource(request),
+		getDataTypeForResource(request.Resource),
+		request.Resource.String(),
+	)
+
+	r := s.connection.R()
+	resp, err := r.Execute(resty.MethodGet, path)
+	if err != nil {
+		return &ResponseFiles{Response: Response{R: *request, Error: fmt.Errorf("failed to send request: %v", err)}}
+	}
+
+	if resp.IsError() {
+		return &ResponseFiles{Response: Response{R: *request, Error: fmt.Errorf("request failed with status: %v", resp.Status())}}
+	}
+
+	filename := request.Resource.String()
+	if request.CustomFilename != "" {
+		filename = request.CustomFilename
+	}
+
+	filepath := filepath.Join(request.BaseFolder, filename)
+	err = os.WriteFile(filepath, resp.Body(), 0644)
+	if err != nil {
+		return &ResponseFiles{Response: Response{R: *request, Error: fmt.Errorf("failed to write file: %v", err)}}
+	}
+
+	responseFiles := NewResponseFiles(*request)
+	responseFiles.fileNames = []string{filename}
+	responseFiles.Size = uint64(len(resp.Body()))
+	return responseFiles
+}
+
+func (s *Connector) Upload(request *Request) *Response {
+	if len(request.Files) == 0 {
+		return &Response{R: *request, Error: fmt.Errorf("no files to upload")}
+	}
+
+	path := fmt.Sprintf("/storage/upload?id=%d&dataType=%s&filepath=%s",
+		getIDForResource(request),
+		getDataTypeForResource(request.Resource),
+		request.Resource.String(),
+	)
+
+	r := s.connection.R()
+	for filename, reader := range request.Files {
+		r.SetFileReader("file", filename, reader)
+	}
+
+	resp, err := r.Execute(resty.MethodPost, path)
+	if err != nil {
+		return &Response{R: *request, Error: fmt.Errorf("failed to send request: %v", err)}
+	}
+
+	if resp.IsError() {
+		return &Response{R: *request, Error: fmt.Errorf("request failed with status: %s, body: %s", resp.Status(), resp.String())}
+	}
+
+	return &Response{R: *request}
+}
+
+func (s *Connector) Delete(request *Request) *Response {
+	path := fmt.Sprintf("/storage/remove?id=%d&dataType=%s&filepath=%s",
+		getIDForResource(request),
+		getDataTypeForResource(request.Resource),
+		request.Resource.String(),
+	)
+
+	r := s.connection.R()
+	resp, err := r.Execute(resty.MethodDelete, path)
+	if err != nil {
+		return &Response{R: *request, Error: fmt.Errorf("failed to send request: %v", err)}
+	}
+
+	if resp.IsError() {
+		return &Response{R: *request, Error: fmt.Errorf("request failed with status: %s, body: %s", resp.Status(), resp.String())}
+	}
+
+	return &Response{R: *request}
+}
+
+func getIDForResource(request *Request) uint64 {
+	switch request.Resource {
+	case resource.SourceCode, resource.CompiledBinary, resource.CompileOutput:
+		return request.SubmitID
+	case resource.Checker, resource.Interactor:
+		return request.ProblemID
+	case resource.Test:
+		if request.TestID > 0 {
+			return request.TestID
+		}
+		return request.ProblemID
+	default:
+		return 0
+	}
+}
+
+func getDataTypeForResource(resourceType resource.Type) string {
+	switch resourceType {
+	case resource.SourceCode, resource.CompiledBinary, resource.CompileOutput:
+		return "submission"
+	case resource.Checker, resource.Interactor, resource.Test:
+		return "problem"
+	default:
+		return "unknown"
+	}
+}