Merge remote-tracking branch 'origin/main' into dev/storage

Author: Artem Titov

common/connectors/storageconn/connector.go

@@ -1,11 +1,11 @@
 package storageconn
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"mime"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing_system/common/config"
@@ -25,9 +25,6 @@ func NewConnector(connection *config.Connection) *Connector {
 
 func (s *Connector) Download(request *Request) *FileResponse {
 	response := NewFileResponse(*request)
-	if request.StorageFilename == "" {
-		response.Error = fmt.Errorf("storage filename not specified")
-	}
 
 	if err := os.MkdirAll(request.BaseFolder, 0775); err != nil {
 		response.Error = fmt.Errorf("failed to create base folder: %v", err)
@@ -47,18 +44,23 @@ func (s *Connector) Download(request *Request) *FileResponse {
 		"request": string(requestJSON),
 	})
 
-	resp, err := r.Get(path)
+	resp, err := r.SetDoNotParseResponse(true).Execute("GET", path)
 	if err != nil {
 		response.Error = fmt.Errorf("failed to send request: %v", err)
 		return response
 	}
+	defer resp.RawBody().Close()
 
-	if resp.IsError() {
-		response.Error = fmt.Errorf("get request failed with status: %v", resp.Status())
+	if resp.StatusCode() >= 400 {
+		if resp.StatusCode() == http.StatusNotFound {
+			response.Error = ErrStorageFileNotFound
+		} else {
+			response.Error = fmt.Errorf("get request failed with status: %v", resp.Status())
+		}
 		return response
 	}
 
-	filename := ""
+	var filename string
 	if request.DownloadFilename != nil && *request.DownloadFilename != "" {
 		filename = *request.DownloadFilename
 	} else {
@@ -85,24 +87,20 @@ func (s *Connector) Download(request *Request) *FileResponse {
 	}
 	defer file.Close()
 
-	_, err = io.Copy(file, bytes.NewReader(resp.Body()))
+	written, err := io.Copy(file, resp.RawBody())
 	if err != nil {
 		response.Error = fmt.Errorf("failed to write to file: %v", err)
 		return response
 	}
 
 	response.Filename = filename
 	response.BaseFolder = request.BaseFolder
-	response.Size = uint64(len(resp.Body()))
+	response.Size = uint64(written)
 	return response
 }
 
 func (s *Connector) Upload(request *Request) *Response {
 	response := &Response{R: *request}
-	if request.StorageFilename == "" {
-		response.Error = fmt.Errorf("storage filename not specified")
-		return response
-	}
 
 	if request.File == nil {
 		response.Error = fmt.Errorf("file for upload is not specified")
@@ -121,6 +119,8 @@ func (s *Connector) Upload(request *Request) *Response {
 	r.SetFormData(map[string]string{
 		"request": string(requestJSON),
 	})
+
+	// request.StorageFilename can be empty
 	r.SetFileReader("file", request.StorageFilename, request.File)
 
 	resp, err := r.Post(path)
@@ -139,10 +139,6 @@ func (s *Connector) Upload(request *Request) *Response {
 
 func (s *Connector) Delete(request *Request) *Response {
 	response := &Response{R: *request}
-	if request.StorageFilename == "" {
-		response.Error = fmt.Errorf("storage filename not specified")
-		return response
-	}
 
 	path := "/storage/remove"
 	r := s.connection.R()
@@ -153,7 +149,7 @@ func (s *Connector) Delete(request *Request) *Response {
 		return response
 	}
 
-	r.SetQueryParams(map[string]string{
+	r.SetFormData(map[string]string{
 		"request": string(requestJSON),
 	})
 
@@ -170,4 +166,3 @@ func (s *Connector) Delete(request *Request) *Response {
 
 	return response
 }
-

common/connectors/storageconn/errors.go

@@ -0,0 +1,5 @@
+package storageconn
+
+import "errors"
+
+var ErrStorageFileNotFound = errors.New("file not found in storage")

common/connectors/storageconn/structs.go

@@ -34,7 +34,7 @@ type Request struct {
 	// For uploads, File should be specified
 	File io.Reader `json:"-"`
 
-	// StorageFilename should be always specified
+	// If StorageFilename is not specified, Storage tries to get the filename automatically
 	StorageFilename string `json:"storageFilename"`
 }
 

common/constants/resource/resource_type.go

@@ -17,6 +17,7 @@ const (
 	CheckerOutput
 	Interactor
 	// Will be increased
+	// Don't forget to add a new type to storage/filesystem/resource_info.go
 )
 
 type DataType int

main.go

@@ -34,7 +34,7 @@ func main() {
 	if ts.Config.Storage != nil {
 		err := storage.SetupStorage(ts)
 		if err != nil {
-			logger.Panic(err.Error())
+			logger.Panic("Can not setup storage, error: %v", err.Error())
 		}
 	} else {
 		logger.Info("storage is not configured, skipping storage component")

storage/filesystem/filesystem.go

@@ -5,7 +5,7 @@ import (
 	"mime/multipart"
 	"os"
 	"path/filepath"
-	"strings"
+	"strconv"
 	"testing_system/common/config"
 	"testing_system/lib/logger"
 
@@ -21,25 +21,24 @@ func NewFilesystem(storageConfig *config.StorageConfig) IFilesystem {
 	return &Filesystem{Basepath: storageConfig.StoragePath, BlockSize: storageConfig.BlockSize}
 }
 
-func (filesystem *Filesystem) generatePathFromID(prefix string, id string) string {
-	var parts []string
-	parts = append(parts, prefix)
+func (filesystem *Filesystem) generatePathFromID(id string) string {
+	parts := []string{}
 	for i := 0; i < len(id); i += int(filesystem.BlockSize) {
 		end := min(i+int(filesystem.BlockSize), len(id))
 		parts = append(parts, id[i:end])
 	}
 	return filepath.Join(parts...)
 }
 
-func (filesystem *Filesystem) SaveFile(c *gin.Context, prefix string, id string, filename string, file *multipart.FileHeader) error {
-	fullPath, err := filesystem.GetFilePath(prefix, id, filename)
+func (filesystem *Filesystem) SaveFile(c *gin.Context, resourceInfo *ResourceInfo, file *multipart.FileHeader) error {
+	fullPath, err := filesystem.BuildFilePath(resourceInfo)
 	if err != nil {
 		return err
 	}
 
-	fullDirPath := filepath.Dir(fullPath)
-	if err := os.MkdirAll(fullDirPath, 0755); err != nil {
-		return fmt.Errorf("failed to create directory: %w", err)
+	dir := filepath.Dir(fullPath)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("failed to create directory %s: %w", dir, err)
 	}
 
 	if _, err := os.Stat(fullPath); err == nil {
@@ -49,8 +48,8 @@ func (filesystem *Filesystem) SaveFile(c *gin.Context, prefix string, id string,
 	return c.SaveUploadedFile(file, fullPath)
 }
 
-func (filesystem *Filesystem) RemoveFile(prefix string, id string, filename string) error {
-	fullPath, err := filesystem.GetFilePath(prefix, id, filename)
+func (filesystem *Filesystem) RemoveFile(resourceInfo *ResourceInfo) error {
+	fullPath, err := filesystem.BuildFilePath(resourceInfo)
 	if err != nil {
 		return err
 	}
@@ -60,16 +59,18 @@ func (filesystem *Filesystem) RemoveFile(prefix string, id string, filename stri
 	}
 
 	dir := filepath.Dir(fullPath)
-	basePath, _ := filepath.Abs(filesystem.Basepath)
 
 	for {
-		absDir, _ := filepath.Abs(dir)
-		if absDir == basePath || !strings.HasPrefix(absDir, basePath) {
+		if dir == filesystem.Basepath {
 			break
 		}
 
 		entries, err := os.ReadDir(dir)
-		if err != nil || len(entries) > 0 {
+
+		if err != nil {
+			return fmt.Errorf("failed to read directory %s: %w", dir, err)
+		}
+		if len(entries) > 0 {
 			break
 		}
 
@@ -82,28 +83,34 @@ func (filesystem *Filesystem) RemoveFile(prefix string, id string, filename stri
 	return nil
 }
 
-func (filesystem *Filesystem) GetFilePath(prefix, id, filename string) (string, error) {
-	if prefix == "" || id == "" || filename == "" {
-		return "", fmt.Errorf("prefix, id, and filename cannot be empty")
-	}
-
-	cleanFilename := filepath.Base(filename)
-
-	subpath := filesystem.generatePathFromID(prefix, id)
-	fullPath := filepath.Join(filesystem.Basepath, subpath, cleanFilename)
-
-	absBasepath, err := filepath.Abs(filesystem.Basepath)
+func (filesystem *Filesystem) GetFile(resourceInfo *ResourceInfo) (string, error) {
+	fullPath, err := filesystem.BuildFilePath(resourceInfo)
 	if err != nil {
 		return "", err
 	}
 
-	absFullPath, err := filepath.Abs(fullPath)
+	_, err = os.Stat(fullPath)
+
 	if err != nil {
 		return "", err
 	}
 
-	if !strings.HasPrefix(absFullPath, absBasepath) {
-		return "", fmt.Errorf("invalid path: basepath is not a prefix of fullpath")
+	return fullPath, nil
+}
+
+func (filesystem *Filesystem) BuildFilePath(resourceInfo *ResourceInfo) (string, error) {
+	subpathID := filesystem.generatePathFromID(strconv.FormatUint(resourceInfo.ID, 10))
+	fullPath := filepath.Join(filesystem.Basepath, resourceInfo.DataType.String(), subpathID, resourceInfo.Filepath)
+
+	if resourceInfo.EmptyStorageFilename {
+		entries, err := os.ReadDir(fullPath)
+		if err != nil {
+			return "", fmt.Errorf("failed to read directory %s: %v", fullPath, err)
+		}
+		if len(entries) != 1 || entries[0].IsDir() {
+			return "", fmt.Errorf("StorageFilename is not specified, but directory %s does not contain exactly one file", fullPath)
+		}
+		fullPath = filepath.Join(fullPath, entries[0].Name())
 	}
 
 	return fullPath, nil

storage/filesystem/interface.go

@@ -7,7 +7,7 @@ import (
 )
 
 type IFilesystem interface {
-	SaveFile(c *gin.Context, prefix string, id string, filename string, file *multipart.FileHeader) error
-	RemoveFile(prefix string, id string, filename string) error
-	GetFilePath(prefix string, id string, filename string) (string, error)
+	SaveFile(c *gin.Context, resourceInfo *ResourceInfo, file *multipart.FileHeader) error
+	RemoveFile(resourceInfo *ResourceInfo) error
+	GetFile(resourceInfo *ResourceInfo) (string, error)
 }