Merge branch 'main' into dev/storage

Author: Artyom321

.github/workflows/go.yml

@@ -22,7 +22,9 @@ jobs:
         go-version: '1.24'
 
     - name: Build
-      run: go build -v ./...
+      run: |
+        go generate ./...
+        go build -v ./...
 
     - name: Test
       run: go test -v ./...

.gitignore

@@ -1,2 +1,4 @@
 .idea
-testing_system
+testing_system
+cp.sh
+swag

common/config/invoker.go

@@ -1,21 +1,28 @@
 package config
 
+import "time"
+
 type InvokerConfig struct {
+	// PublicAddress defines address for public access to invoker from master if the server is set up locally with some proxy
+	PublicAddress *string `yaml:"PublicAddress,omitempty"`
+
 	Threads   uint64 `yaml:"Threads"`
 	Sandboxes uint64 `yaml:"Sandboxes"`
 	QueueSize uint64 `yaml:"QueueSize"`
 
 	SandboxType     string `yaml:"SandboxType"`
 	SandboxHomePath string `yaml:"SandboxHomePath"`
 
+	MasterPingInterval time.Duration `yaml:"MasterPingInterval"`
+
 	CacheSize uint64 `yaml:"CacheSize"`
 	CachePath string `yaml:"CachePath"`
 
 	SaveOutputHead *uint64 `yaml:"SaveOutputHead,omitempty"`
 
 	CompilerConfigsFolder string `yaml:"CompilerConfigsFolder"`
 
-	CheckerLimits *RunConfig `yaml:"CheckerLimits,omitempty"`
+	CheckerLimits *RunLimitsConfig `yaml:"CheckerLimits,omitempty"`
 }
 
 func FillInInvokerConfig(config *InvokerConfig) {
@@ -36,10 +43,15 @@ func FillInInvokerConfig(config *InvokerConfig) {
 		switch config.SandboxType {
 		case "simple":
 			panic("No sandbox home path specified")
+		case "isolate":
+			panic("No isolate home path specified (it is used for meta files)")
 		default:
 			panic("unsupported sandbox type: " + config.SandboxType)
 		}
 	}
+	if config.MasterPingInterval == 0 {
+		config.MasterPingInterval = time.Second * 10
+	}
 	if len(config.CachePath) == 0 {
 		panic("No invoker cache path specified")
 	}
@@ -51,7 +63,7 @@ func FillInInvokerConfig(config *InvokerConfig) {
 	}
 
 	if config.CheckerLimits == nil {
-		config.CheckerLimits = &RunConfig{}
+		config.CheckerLimits = &RunLimitsConfig{}
 	}
-	fillInDefaultCheckerRunConfig(config.CheckerLimits)
+	fillInDefaultCheckerRunLimitsConfig(config.CheckerLimits)
 }

common/config/run_config.go

@@ -0,0 +1,40 @@
+package config
+
+import "testing_system/lib/customfields"
+
+type RunLimitsConfig struct {
+	TimeLimit     customfields.Time   `yaml:"TimeLimit" json:"TimeLimit"`
+	MemoryLimit   customfields.Memory `yaml:"MemoryLimit" json:"MemoryLimit"`
+	WallTimeLimit customfields.Time   `yaml:"WallTimeLimit" json:"WallTimeLimit"`
+
+	// MaxThreads specifies max number of threads and/or processes.
+	// If MaxThreads is unspecified (equals 0) it is considered as 1.
+	// If MaxThreads = -1, any number of threads is allowed
+	MaxThreads int64 `yaml:"MaxThreads" json:"MaxThreads"`
+
+	// MaxOpenFiles specifies max number of files, opened by process
+	// If MaxOpenFiles is unspecified (equals 0), it is considered as 64
+	MaxOpenFiles uint64 `yaml:"MaxOpenFiles" json:"MaxOpenFiles"`
+
+	// MaxOutputSize specifies max output in EACH file.
+	// If MaxOpenFiles is unspecified (equals 0), it is considered as 1g
+	MaxOutputSize customfields.Memory `yaml:"MaxOutputSize" json:"MaxOutputSize"`
+}
+
+func fillInDefaultCheckerRunLimitsConfig(config *RunLimitsConfig) {
+	if config.TimeLimit == 0 {
+		config.TimeLimit.FromStr("15s")
+	}
+	if config.MemoryLimit == 0 {
+		config.MemoryLimit.FromStr("1g")
+	}
+	if config.WallTimeLimit == 0 {
+		config.WallTimeLimit.FromStr("30s")
+	}
+	if config.MaxOpenFiles == 0 {
+		config.MaxOpenFiles = 64
+	}
+	if config.MaxOutputSize == 0 {
+		config.MaxOutputSize.FromStr("1g")
+	}
+}

common/config/run_limits_config.go

@@ -1,10 +1,11 @@
 package invokerconn
 
 import (
-	"github.com/go-resty/resty/v2"
 	"testing_system/common/config"
 	"testing_system/common/connectors"
 	"testing_system/lib/connector"
+
+	"github.com/go-resty/resty/v2"
 )
 
 type Connector struct {
@@ -15,15 +16,15 @@ func NewConnector(connection *config.Connection) *Connector {
 	return &Connector{connectors.NewConnectorBase(connection)}
 }
 
-func (c *Connector) Status() (*StatusResponse, error) {
+func (c *Connector) Status() (*Status, error) {
 	r := c.connection.R()
 
-	return connector.Receive[StatusResponse](r, "/invoker/status", resty.MethodGet)
+	return connector.Receive[Status](r, "/invoker/status", resty.MethodGet)
 }
 
-func (c *Connector) NewJob(job *Job) (*StatusResponse, error) {
+func (c *Connector) NewJob(job *Job) (*Status, error) {
 	r := c.connection.R()
 	r.SetBody(job)
 
-	return connector.Receive[StatusResponse](r, "/invoker/job/new", resty.MethodPost)
+	return connector.Receive[Status](r, "/invoker/job/new", resty.MethodPost)
 }

common/connectors/invokerconn/connector.go

@@ -16,9 +16,9 @@ type Job struct {
 	// TODO: Add job dependency
 }
 
-type StatusResponse struct {
-	// TODO: add some information about invoker state
+type Status struct {
 	MaxNewJobs   uint64   `json:"MaxNewJobs"`
 	ActiveJobIDs []string `json:"ActiveJobIDs"`
-	Epoch        int      `json:"Epoch"`
+	Epoch        string   `json:"Epoch"`
+	Address      string   `json:"Address"`
 }

common/connectors/invokerconn/structs.go

@@ -1,15 +1,32 @@
 package masterconn
 
-import "testing_system/common/config"
+import (
+	"testing_system/common/config"
+	"testing_system/common/connectors"
+	"testing_system/common/connectors/invokerconn"
+	"testing_system/lib/connector"
+
+	"github.com/go-resty/resty/v2"
+)
 
 type Connector struct {
-	// TODO: Add master connection
+	connection *connectors.ConnectorBase
 }
 
 func NewConnector(connection *config.Connection) *Connector {
-	return nil
+	return &Connector{connectors.NewConnectorBase(connection)}
 }
 
-func (c *Connector) InvokerJobResult(result *InvokerJobResult) error {
-	return nil
+func (c *Connector) SendInvokerJobResult(result *InvokerJobResult) error {
+	r := c.connection.R()
+	r.SetBody(result)
+
+	return connector.ReceiveEmpty(r, "/master/invoker/job-result", resty.MethodPost)
+}
+
+func (c *Connector) SendInvokerStatus(response *invokerconn.Status) error {
+	r := c.connection.R()
+	r.SetBody(response)
+
+	return connector.ReceiveEmpty(r, "/master/invoker/status", resty.MethodPost)
 }

common/connectors/masterconn/connector.go

@@ -16,7 +16,7 @@ type InvokerJobResult struct {
 
 	Statistics *JobResultStatistics `json:"Statistics"`
 
-	InvokerStatus *invokerconn.StatusResponse `json:"InvokerStatus"`
+	InvokerStatus *invokerconn.Status `json:"InvokerStatus"`
 }
 
 type JobResultStatistics struct {

common/connectors/masterconn/structs.go

@@ -1,7 +1,7 @@
-package resource
+//go:generate go run golang.org/x/tools/cmd/stringer@latest -type=Type
+//go:generate go run golang.org/x/tools/cmd/stringer@latest -type=DataType
 
-//go:generate stringer -type=Type
-//go:generate stringer -type=DataType
+package resource
 
 type Type int