Merge remote-tracking branch 'origin/main' into dev/storage

Author: Artem Titov

common/connectors/storageconn/connector.go

@@ -25,9 +25,6 @@ func NewConnector(connection *config.Connection) *Connector {
 
 func (s *Connector) Download(request *Request) *FileResponse {
 	response := NewFileResponse(*request)
-	if request.StorageFilename == "" {
-		response.Error = fmt.Errorf("storage filename not specified")
-	}
 
 	if err := os.MkdirAll(request.BaseFolder, 0775); err != nil {
 		response.Error = fmt.Errorf("failed to create base folder: %v", err)
@@ -99,10 +96,6 @@ func (s *Connector) Download(request *Request) *FileResponse {
 
 func (s *Connector) Upload(request *Request) *Response {
 	response := &Response{R: *request}
-	if request.StorageFilename == "" {
-		response.Error = fmt.Errorf("storage filename not specified")
-		return response
-	}
 
 	if request.File == nil {
 		response.Error = fmt.Errorf("file for upload is not specified")
@@ -121,6 +114,8 @@ func (s *Connector) Upload(request *Request) *Response {
 	r.SetFormData(map[string]string{
 		"request": string(requestJSON),
 	})
+
+	// request.StorageFilename can be empty
 	r.SetFileReader("file", request.StorageFilename, request.File)
 
 	resp, err := r.Post(path)
@@ -139,10 +134,6 @@ func (s *Connector) Upload(request *Request) *Response {
 
 func (s *Connector) Delete(request *Request) *Response {
 	response := &Response{R: *request}
-	if request.StorageFilename == "" {
-		response.Error = fmt.Errorf("storage filename not specified")
-		return response
-	}
 
 	path := "/storage/remove"
 	r := s.connection.R()
@@ -153,7 +144,7 @@ func (s *Connector) Delete(request *Request) *Response {
 		return response
 	}
 
-	r.SetQueryParams(map[string]string{
+	r.SetFormData(map[string]string{
 		"request": string(requestJSON),
 	})
 

common/connectors/storageconn/structs.go

@@ -34,7 +34,7 @@ type Request struct {
 	// For uploads, File should be specified
 	File io.Reader `json:"-"`
 
-	// StorageFilename should be always specified
+	// If StorageFilename is not specified, Storage tries to get the filename automatically
 	StorageFilename string `json:"storageFilename"`
 }
 

common/constants/resource/resource_type.go

@@ -17,8 +17,37 @@ const (
 	CheckerOutput
 	Interactor
 	// Will be increased
+	// Don't forget to add a new type to storage/filesystem/resource_info.go
 )
 
+// FilepathFolderMapping should contain all types
+var FilepathFolderMapping = map[Type]string{
+	SourceCode:     "source",
+	CompiledBinary: "",
+	CompileOutput:  "",
+	TestInput:      "tests",
+	TestAnswer:     "tests",
+	TestOutput:     "tests",
+	TestStderr:     "tests",
+	Checker:        "checker",
+	CheckerOutput:  "tests",
+	Interactor:     "interactor",
+}
+
+// FilepathFilenameMapping should contain all types
+var FilepathFilenameMapping = map[Type]string{
+	SourceCode:     "source",
+	CompiledBinary: "solution",
+	CompileOutput:  "compile.out",
+	TestInput:      "%02d",
+	TestAnswer:     "%02d.a",
+	TestOutput:     "%02d.out",
+	TestStderr:     "%02d.err",
+	Checker:        "checker",
+	CheckerOutput:  "%02d.check",
+	Interactor:     "interactor",
+}
+
 type DataType int
 
 const (

main.go

@@ -34,7 +34,7 @@ func main() {
 	if ts.Config.Storage != nil {
 		err := storage.SetupStorage(ts)
 		if err != nil {
-			logger.Panic(err.Error())
+			logger.Panic("Can not setup storage, error: %v", err.Error())
 		}
 	} else {
 		logger.Info("storage is not configured, skipping storage component")

storage/filesystem/filesystem.go

@@ -5,6 +5,7 @@ import (
 	"mime/multipart"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing_system/common/config"
 	"testing_system/lib/logger"
@@ -22,24 +23,34 @@ func NewFilesystem(storageConfig *config.StorageConfig) IFilesystem {
 }
 
 func (filesystem *Filesystem) generatePathFromID(prefix string, id string) string {
-	var parts []string
-	parts = append(parts, prefix)
+	parts := []string{prefix}
 	for i := 0; i < len(id); i += int(filesystem.BlockSize) {
 		end := min(i+int(filesystem.BlockSize), len(id))
 		parts = append(parts, id[i:end])
 	}
 	return filepath.Join(parts...)
 }
 
-func (filesystem *Filesystem) SaveFile(c *gin.Context, prefix string, id string, filename string, file *multipart.FileHeader) error {
-	fullPath, err := filesystem.GetFilePath(prefix, id, filename)
+func (filesystem *Filesystem) SaveFile(c *gin.Context, resourceInfo *ResourceInfo, file *multipart.FileHeader) error {
+	fullPath, err := filesystem.BuildFilePath(resourceInfo)
 	if err != nil {
 		return err
 	}
 
-	fullDirPath := filepath.Dir(fullPath)
-	if err := os.MkdirAll(fullDirPath, 0755); err != nil {
-		return fmt.Errorf("failed to create directory: %w", err)
+	dir := filepath.Dir(fullPath)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("failed to create directory %s: %w", dir, err)
+	}
+
+	entries, err := filesystem.GetDirEntriesNames(dir)
+	if err != nil {
+		return err
+	}
+
+	if resourceInfo.EmptyStorageFilename {
+		if len(entries) > 1 || (len(entries) == 1 && entries[0] != filepath.Base(resourceInfo.Filepath)) {
+			return fmt.Errorf("failed to save file: directory %s already contains some files different from default filename, but StorageFilename is not specified", dir)
+		}
 	}
 
 	if _, err := os.Stat(fullPath); err == nil {
@@ -49,17 +60,28 @@ func (filesystem *Filesystem) SaveFile(c *gin.Context, prefix string, id string,
 	return c.SaveUploadedFile(file, fullPath)
 }
 
-func (filesystem *Filesystem) RemoveFile(prefix string, id string, filename string) error {
-	fullPath, err := filesystem.GetFilePath(prefix, id, filename)
+func (filesystem *Filesystem) RemoveFile(resourceInfo *ResourceInfo) error {
+	fullPath, err := filesystem.BuildFilePath(resourceInfo)
+	if err != nil {
+		return err
+	}
+
+	dir := filepath.Dir(fullPath)
+	entries, err := filesystem.GetDirEntriesNames(dir)
 	if err != nil {
 		return err
 	}
 
+	if resourceInfo.EmptyStorageFilename {
+		if len(entries) > 1 || (len(entries) == 1 && entries[0] != filepath.Base(resourceInfo.Filepath)) {
+			return fmt.Errorf("failed to remove file: directory %s contains some files different from default filename, but StorageFilename is not specified", dir)
+		}
+	}
+
 	if err := os.Remove(fullPath); err != nil {
 		return fmt.Errorf("failed to remove file %s: %w", fullPath, err)
 	}
 
-	dir := filepath.Dir(fullPath)
 	basePath, _ := filepath.Abs(filesystem.Basepath)
 
 	for {
@@ -82,15 +104,38 @@ func (filesystem *Filesystem) RemoveFile(prefix string, id string, filename stri
 	return nil
 }
 
-func (filesystem *Filesystem) GetFilePath(prefix, id, filename string) (string, error) {
-	if prefix == "" || id == "" || filename == "" {
-		return "", fmt.Errorf("prefix, id, and filename cannot be empty")
+func (filesystem *Filesystem) GetFile(resourceInfo *ResourceInfo) (string, error) {
+	fullPath, err := filesystem.BuildFilePath(resourceInfo)
+	if err != nil {
+		return "", err
 	}
 
-	cleanFilename := filepath.Base(filename)
+	dir := filepath.Dir(fullPath)
+	entries, err := filesystem.GetDirEntriesNames(dir)
+	if err != nil {
+		return "", err
+	}
+
+	if resourceInfo.EmptyStorageFilename {
+		if len(entries) > 1 || (len(entries) == 1 && entries[0] != filepath.Base(resourceInfo.Filepath)) {
+			return "", fmt.Errorf("failed to get file: directory %s contains some files different from default filename, but StorageFilename is not specified", dir)
+		}
+	}
+
+	_, err = os.Stat(fullPath)
 
-	subpath := filesystem.generatePathFromID(prefix, id)
-	fullPath := filepath.Join(filesystem.Basepath, subpath, cleanFilename)
+	if err != nil {
+		return "", err
+	}
+
+	return fullPath, nil
+}
+
+func (filesystem *Filesystem) BuildFilePath(resourceInfo *ResourceInfo) (string, error) {
+	cleanFilename := filepath.Base(resourceInfo.Filepath)
+
+	subpathID := filesystem.generatePathFromID(resourceInfo.DataType.String(), strconv.FormatUint(resourceInfo.ID, 10))
+	fullPath := filepath.Join(filesystem.Basepath, subpathID, cleanFilename)
 
 	absBasepath, err := filepath.Abs(filesystem.Basepath)
 	if err != nil {
@@ -103,8 +148,23 @@ func (filesystem *Filesystem) GetFilePath(prefix, id, filename string) (string,
 	}
 
 	if !strings.HasPrefix(absFullPath, absBasepath) {
-		return "", fmt.Errorf("invalid path: basepath is not a prefix of fullpath")
+		return "", fmt.Errorf("invalid path: filesystem basepath is not a prefix of file fullpath")
 	}
 
 	return fullPath, nil
 }
+
+func (filesystem *Filesystem) GetDirEntriesNames(dir string) ([]string, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read directory %s: %v", dir, err)
+	}
+
+	names := make([]string, 0, len(entries))
+
+	for _, entry := range entries {
+		names = append(names, entry.Name())
+	}
+
+	return names, nil
+}

storage/filesystem/interface.go

@@ -7,7 +7,7 @@ import (
 )
 
 type IFilesystem interface {
-	SaveFile(c *gin.Context, prefix string, id string, filename string, file *multipart.FileHeader) error
-	RemoveFile(prefix string, id string, filename string) error
-	GetFilePath(prefix string, id string, filename string) (string, error)
+	SaveFile(c *gin.Context, resourceInfo *ResourceInfo, file *multipart.FileHeader) error
+	RemoveFile(resourceInfo *ResourceInfo) error
+	GetFile(resourceInfo *ResourceInfo) (string, error)
 }

storage/filesystem/resource_info.go

@@ -0,0 +1,113 @@
+package filesystem
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"testing_system/common/connectors/storageconn"
+	"testing_system/common/constants/resource"
+)
+
+type ResourceInfo struct {
+	Request              *storageconn.Request
+	ID                   uint64
+	Filepath             string
+	DataType             resource.DataType
+	EmptyStorageFilename bool
+}
+
+func (resourseInfo *ResourceInfo) ParseDataType() error {
+	request := resourseInfo.Request
+
+	switch request.Resource {
+	case resource.Checker, resource.Interactor:
+		resourseInfo.DataType = resource.Problem
+		return nil
+	case resource.TestInput, resource.TestAnswer:
+		resourseInfo.DataType = resource.Problem
+		return nil
+	case resource.SourceCode, resource.CompiledBinary, resource.CompileOutput:
+		resourseInfo.DataType = resource.Submission
+		return nil
+	case resource.TestOutput, resource.TestStderr, resource.CheckerOutput:
+		resourseInfo.DataType = resource.Submission
+		return nil
+	default:
+		return errors.New("unknown resource type")
+	}
+}
+
+func (resourseInfo *ResourceInfo) ParseDataID() error {
+	request := resourseInfo.Request
+
+	switch resourseInfo.DataType {
+	case resource.Problem:
+		if request.ProblemID == 0 {
+			return errors.New("ProblemID is not specified for promlem resource")
+		}
+		resourseInfo.ID = request.ProblemID
+		return nil
+	case resource.Submission:
+		if request.SubmitID == 0 {
+			return errors.New("SubmitID is not specified for submission resource")
+		}
+		resourseInfo.ID = request.SubmitID
+		return nil
+	default:
+		return errors.New("unavailable to get data id")
+	}
+}
+
+func (resourseInfo *ResourceInfo) ParseFilepath() error {
+	filepathFolder, err := resourseInfo.parseFilepathFolder()
+
+	if err != nil {
+		return fmt.Errorf("unavailable to parse filepathFolder: %v", err)
+	}
+
+	filepathFilename, err := resourseInfo.parseFilepathFilename()
+
+	if err != nil {
+		return fmt.Errorf("unavailable to parse filepathFilename: %v", err)
+	}
+
+	resourseInfo.Filepath = filepath.Join(filepathFolder, filepathFilename)
+
+	return nil
+}
+
+func (resourseInfo *ResourceInfo) parseFilepathFolder() (string, error) {
+	request := resourseInfo.Request
+
+	filepathFolder, ok := resource.FilepathFolderMapping[request.Resource]
+	if !ok {
+		return "", fmt.Errorf("there is no type %s in FilepathFolderMapping", request.Resource.String())
+	}
+
+	return filepathFolder, nil
+}
+
+func (resourseInfo *ResourceInfo) parseFilepathFilename() (string, error) {
+	request := resourseInfo.Request
+
+	if request.StorageFilename != "" {
+		return request.StorageFilename, nil
+	}
+
+	resourseInfo.EmptyStorageFilename = true
+
+	filepathFilename, ok := resource.FilepathFilenameMapping[request.Resource]
+	if !ok {
+		return "", fmt.Errorf("there is no type %s in FilepathFilenameMapping", request.Resource.String())
+	}
+
+	switch request.Resource {
+	case resource.TestInput, resource.TestAnswer, resource.TestOutput, resource.TestStderr, resource.CheckerOutput:
+		if request.TestID == 0 {
+			return "", errors.New("TestID is not specified for test resource")
+		}
+		return fmt.Sprintf(filepathFilename, request.TestID), nil
+	default:
+		return filepathFilename, nil
+	}
+}