Merge remote-tracking branch 'origin/main' into dev/storage

Author: Artem Titov

common/connectors/storageconn/connector.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"mime"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing_system/common/config"
@@ -25,9 +26,6 @@ func NewConnector(connection *config.Connection) *Connector {
 
 func (s *Connector) Download(request *Request) *FileResponse {
 	response := NewFileResponse(*request)
-	if request.StorageFilename == "" {
-		response.Error = fmt.Errorf("storage filename not specified")
-	}
 
 	if err := os.MkdirAll(request.BaseFolder, 0775); err != nil {
 		response.Error = fmt.Errorf("failed to create base folder: %v", err)
@@ -54,7 +52,11 @@ func (s *Connector) Download(request *Request) *FileResponse {
 	}
 
 	if resp.IsError() {
-		response.Error = fmt.Errorf("get request failed with status: %v", resp.Status())
+		if resp.StatusCode() == http.StatusNotFound {
+			response.Error = ErrStorageFileNotFound
+		} else {
+			response.Error = fmt.Errorf("get request failed with status: %v", resp.Status())
+		}
 		return response
 	}
 
@@ -99,10 +101,6 @@ func (s *Connector) Download(request *Request) *FileResponse {
 
 func (s *Connector) Upload(request *Request) *Response {
 	response := &Response{R: *request}
-	if request.StorageFilename == "" {
-		response.Error = fmt.Errorf("storage filename not specified")
-		return response
-	}
 
 	if request.File == nil {
 		response.Error = fmt.Errorf("file for upload is not specified")
@@ -121,6 +119,8 @@ func (s *Connector) Upload(request *Request) *Response {
 	r.SetFormData(map[string]string{
 		"request": string(requestJSON),
 	})
+
+	// request.StorageFilename can be empty
 	r.SetFileReader("file", request.StorageFilename, request.File)
 
 	resp, err := r.Post(path)
@@ -139,10 +139,6 @@ func (s *Connector) Upload(request *Request) *Response {
 
 func (s *Connector) Delete(request *Request) *Response {
 	response := &Response{R: *request}
-	if request.StorageFilename == "" {
-		response.Error = fmt.Errorf("storage filename not specified")
-		return response
-	}
 
 	path := "/storage/remove"
 	r := s.connection.R()
@@ -153,7 +149,7 @@ func (s *Connector) Delete(request *Request) *Response {
 		return response
 	}
 
-	r.SetQueryParams(map[string]string{
+	r.SetFormData(map[string]string{
 		"request": string(requestJSON),
 	})
 
@@ -170,4 +166,3 @@ func (s *Connector) Delete(request *Request) *Response {
 
 	return response
 }
-

common/connectors/storageconn/errors.go

@@ -0,0 +1,5 @@
+package storageconn
+
+import "errors"
+
+var ErrStorageFileNotFound = errors.New("file not found in storage")

common/connectors/storageconn/structs.go

@@ -34,7 +34,7 @@ type Request struct {
 	// For uploads, File should be specified
 	File io.Reader `json:"-"`
 
-	// StorageFilename should be always specified
+	// If StorageFilename is not specified, Storage tries to get the filename automatically
 	StorageFilename string `json:"storageFilename"`
 }
 

common/constants/resource/resource_type.go

@@ -17,8 +17,33 @@ const (
 	CheckerOutput
 	Interactor
 	// Will be increased
+	// Don't forget to add a new type to storage/filesystem/resource_info.go
 )
 
+// FilepathFolderMapping should contain all types
+var FilepathFolderMapping = map[Type]string{
+	SourceCode:     "source",
+	CompiledBinary: "",
+	CompileOutput:  "",
+	TestInput:      "tests",
+	TestAnswer:     "tests",
+	TestOutput:     "tests",
+	TestStderr:     "tests",
+	Checker:        "checker",
+	CheckerOutput:  "tests",
+	Interactor:     "interactor",
+}
+
+var FilepathFilenameMapping = map[Type]string{
+	CompiledBinary: "solution",
+	CompileOutput:  "compile.out",
+	TestInput:      "%02d",
+	TestAnswer:     "%02d.a",
+	TestOutput:     "%02d.out",
+	TestStderr:     "%02d.err",
+	CheckerOutput:  "%02d.check",
+}
+
 type DataType int
 
 const (

main.go

@@ -34,7 +34,7 @@ func main() {
 	if ts.Config.Storage != nil {
 		err := storage.SetupStorage(ts)
 		if err != nil {
-			logger.Panic(err.Error())
+			logger.Panic("Can not setup storage, error: %v", err.Error())
 		}
 	} else {
 		logger.Info("storage is not configured, skipping storage component")

storage/filesystem/filesystem.go

@@ -5,7 +5,7 @@ import (
 	"mime/multipart"
 	"os"
 	"path/filepath"
-	"strings"
+	"strconv"
 	"testing_system/common/config"
 	"testing_system/lib/logger"
 
@@ -21,25 +21,24 @@ func NewFilesystem(storageConfig *config.StorageConfig) IFilesystem {
 	return &Filesystem{Basepath: storageConfig.StoragePath, BlockSize: storageConfig.BlockSize}
 }
 
-func (filesystem *Filesystem) generatePathFromID(prefix string, id string) string {
-	var parts []string
-	parts = append(parts, prefix)
+func (filesystem *Filesystem) generatePathFromID(id string) string {
+	parts := []string{}
 	for i := 0; i < len(id); i += int(filesystem.BlockSize) {
 		end := min(i+int(filesystem.BlockSize), len(id))
 		parts = append(parts, id[i:end])
 	}
 	return filepath.Join(parts...)
 }
 
-func (filesystem *Filesystem) SaveFile(c *gin.Context, prefix string, id string, filename string, file *multipart.FileHeader) error {
-	fullPath, err := filesystem.GetFilePath(prefix, id, filename)
+func (filesystem *Filesystem) SaveFile(c *gin.Context, resourceInfo *ResourceInfo, file *multipart.FileHeader) error {
+	fullPath, err := filesystem.BuildFilePath(resourceInfo)
 	if err != nil {
 		return err
 	}
 
-	fullDirPath := filepath.Dir(fullPath)
-	if err := os.MkdirAll(fullDirPath, 0755); err != nil {
-		return fmt.Errorf("failed to create directory: %w", err)
+	dir := filepath.Dir(fullPath)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("failed to create directory %s: %w", dir, err)
 	}
 
 	if _, err := os.Stat(fullPath); err == nil {
@@ -49,8 +48,8 @@ func (filesystem *Filesystem) SaveFile(c *gin.Context, prefix string, id string,
 	return c.SaveUploadedFile(file, fullPath)
 }
 
-func (filesystem *Filesystem) RemoveFile(prefix string, id string, filename string) error {
-	fullPath, err := filesystem.GetFilePath(prefix, id, filename)
+func (filesystem *Filesystem) RemoveFile(resourceInfo *ResourceInfo) error {
+	fullPath, err := filesystem.BuildFilePath(resourceInfo)
 	if err != nil {
 		return err
 	}
@@ -60,16 +59,18 @@ func (filesystem *Filesystem) RemoveFile(prefix string, id string, filename stri
 	}
 
 	dir := filepath.Dir(fullPath)
-	basePath, _ := filepath.Abs(filesystem.Basepath)
 
 	for {
-		absDir, _ := filepath.Abs(dir)
-		if absDir == basePath || !strings.HasPrefix(absDir, basePath) {
+		if dir == filesystem.Basepath {
 			break
 		}
 
 		entries, err := os.ReadDir(dir)
-		if err != nil || len(entries) > 0 {
+
+		if err != nil {
+			return fmt.Errorf("failed to read directory %s: %w", dir, err)
+		}
+		if len(entries) > 0 {
 			break
 		}
 
@@ -82,28 +83,35 @@ func (filesystem *Filesystem) RemoveFile(prefix string, id string, filename stri
 	return nil
 }
 
-func (filesystem *Filesystem) GetFilePath(prefix, id, filename string) (string, error) {
-	if prefix == "" || id == "" || filename == "" {
-		return "", fmt.Errorf("prefix, id, and filename cannot be empty")
+func (filesystem *Filesystem) GetFile(resourceInfo *ResourceInfo) error {
+	fullPath, err := filesystem.BuildFilePath(resourceInfo)
+	if err != nil {
+		return err
 	}
 
-	cleanFilename := filepath.Base(filename)
-
-	subpath := filesystem.generatePathFromID(prefix, id)
-	fullPath := filepath.Join(filesystem.Basepath, subpath, cleanFilename)
+	_, err = os.Stat(fullPath)
 
-	absBasepath, err := filepath.Abs(filesystem.Basepath)
 	if err != nil {
-		return "", err
+		return err
 	}
 
-	absFullPath, err := filepath.Abs(fullPath)
-	if err != nil {
-		return "", err
-	}
+	return nil
+}
 
-	if !strings.HasPrefix(absFullPath, absBasepath) {
-		return "", fmt.Errorf("invalid path: basepath is not a prefix of fullpath")
+func (filesystem *Filesystem) BuildFilePath(resourceInfo *ResourceInfo) (string, error) {
+	subpathID := filesystem.generatePathFromID(strconv.FormatUint(resourceInfo.ID, 10))
+	fullPath := filepath.Join(filesystem.Basepath, resourceInfo.DataType.String(), subpathID, resourceInfo.Filepath)
+
+	if resourceInfo.EmptyStorageFilename {
+		entries, err := os.ReadDir(fullPath)
+		if err != nil {
+			return "", fmt.Errorf("failed to read directory %s: %v", fullPath, err)
+		}
+		if len(entries) != 1 || entries[0].IsDir() {
+			return "", fmt.Errorf("StorageFilename is not specified, but directory %s does not contain exactly one file", fullPath)
+		}
+		fullPath = filepath.Join(fullPath, entries[0].Name())
+		resourceInfo.Filepath = fullPath // It lets use resourceInfo.Filepath after successful SaveFile, RemoveFile and GetFile calls
 	}
 
 	return fullPath, nil

storage/filesystem/interface.go

@@ -7,7 +7,7 @@ import (
 )
 
 type IFilesystem interface {
-	SaveFile(c *gin.Context, prefix string, id string, filename string, file *multipart.FileHeader) error
-	RemoveFile(prefix string, id string, filename string) error
-	GetFilePath(prefix string, id string, filename string) (string, error)
+	SaveFile(c *gin.Context, resourceInfo *ResourceInfo, file *multipart.FileHeader) error
+	RemoveFile(resourceInfo *ResourceInfo) error
+	GetFile(resourceInfo *ResourceInfo) error
 }

storage/filesystem/resource_info.go

@@ -0,0 +1,111 @@
+package filesystem
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"testing_system/common/connectors/storageconn"
+	"testing_system/common/constants/resource"
+)
+
+type ResourceInfo struct {
+	Request              *storageconn.Request
+	ID                   uint64
+	Filepath             string
+	DataType             resource.DataType
+	EmptyStorageFilename bool
+}
+
+func (resourseInfo *ResourceInfo) ParseDataType() error {
+	request := resourseInfo.Request
+
+	switch request.Resource {
+	case resource.Checker, resource.Interactor:
+		resourseInfo.DataType = resource.Problem
+		return nil
+	case resource.TestInput, resource.TestAnswer:
+		resourseInfo.DataType = resource.Problem
+		return nil
+	case resource.SourceCode, resource.CompiledBinary, resource.CompileOutput:
+		resourseInfo.DataType = resource.Submission
+		return nil
+	case resource.TestOutput, resource.TestStderr, resource.CheckerOutput:
+		resourseInfo.DataType = resource.Submission
+		return nil
+	default:
+		return errors.New("unknown resource type")
+	}
+}
+
+func (resourseInfo *ResourceInfo) ParseDataID() error {
+	request := resourseInfo.Request
+
+	switch resourseInfo.DataType {
+	case resource.Problem:
+		if request.ProblemID == 0 {
+			return errors.New("ProblemID is not specified for problem resource")
+		}
+		resourseInfo.ID = request.ProblemID
+		return nil
+	case resource.Submission:
+		if request.SubmitID == 0 {
+			return errors.New("SubmitID is not specified for submission resource")
+		}
+		resourseInfo.ID = request.SubmitID
+		return nil
+	default:
+		return errors.New("unavailable to get data id")
+	}
+}
+
+func (resourseInfo *ResourceInfo) ParseFilepath() error {
+	filepathFolder, err := resourseInfo.parseFilepathFolder()
+
+	if err != nil {
+		return fmt.Errorf("unavailable to parse filepathFolder: %v", err)
+	}
+
+	filepathFilename, err := resourseInfo.parseFilepathFilename()
+
+	if err != nil {
+		return fmt.Errorf("unavailable to parse filepathFilename: %v", err)
+	}
+
+	resourseInfo.Filepath = filepath.Join(filepathFolder, filepathFilename)
+
+	return nil
+}
+
+func (resourseInfo *ResourceInfo) parseFilepathFolder() (string, error) {
+	request := resourseInfo.Request
+
+	filepathFolder, ok := resource.FilepathFolderMapping[request.Resource]
+	if !ok {
+		return "", fmt.Errorf("there is no type %s in FilepathFolderMapping", request.Resource.String())
+	}
+
+	return filepathFolder, nil
+}
+
+func (resourseInfo *ResourceInfo) parseFilepathFilename() (string, error) {
+	request := resourseInfo.Request
+
+	if filepathFilename, ok := resource.FilepathFilenameMapping[request.Resource]; ok {
+		switch request.Resource {
+		case resource.TestInput, resource.TestAnswer, resource.TestOutput, resource.TestStderr, resource.CheckerOutput:
+			if request.TestID == 0 {
+				return "", errors.New("TestID is not specified for test resource")
+			}
+			return fmt.Sprintf(filepathFilename, request.TestID), nil
+		default:
+			return filepathFilename, nil
+		}
+	}
+
+	if request.StorageFilename != "" {
+		return request.StorageFilename, nil
+	}
+	resourseInfo.EmptyStorageFilename = true
+
+	return "", nil
+}