Merge pull request #8 from MichailKon/dev/storage

Dev/storage

Author: Artyom321

common/config/config.go

@@ -16,6 +16,7 @@ type Config struct {
 
 	Invoker *InvokerConfig `yaml:"Invoker,omitempty"`
 	Master  *MasterConfig  `yaml:"Master,omitempty"`
+	Storage *StorageConfig `yaml:"Storage,omitempty"`
 	// TODO: Add instances here
 
 	DB DBConfig `yaml:"DB"`
@@ -48,5 +49,6 @@ func fillInConfig(config *Config) {
 
 	fillInConnections(config)
 	fillInMasterConfig(config.Master)
+	fillInStorageConfig(config.Storage)
 	FillInInvokerConfig(config.Invoker)
 }

common/config/storage.go

@@ -0,0 +1,17 @@
+package config
+
+type StorageConfig struct {
+	StoragePath string `yaml:"StoragePath"`
+
+	BlockSize uint `yaml:"BlockSize"`
+}
+
+func fillInStorageConfig(config *StorageConfig) {
+	if len(config.StoragePath) == 0 {
+		panic("No storage path specified")
+	}
+
+	if config.BlockSize == 0 {
+		config.BlockSize = 3
+	}
+}

common/connectors/storageconn/connector.go

@@ -1,25 +1,168 @@
 package storageconn
 
 import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"os"
+	"path/filepath"
 	"testing_system/common/config"
+	"testing_system/common/connectors"
 )
 
 type Connector struct {
-	// TODO: Add storage data loader
+	connection *connectors.ConnectorBase
 }
 
 func NewConnector(connection *config.Connection) *Connector {
-	return nil
+	if connection == nil {
+		return nil
+	}
+	return &Connector{connectors.NewConnectorBase(connection)}
 }
 
-func (s *Connector) Download(request *Request) *ResponseFiles {
-	return nil
+func (s *Connector) Download(request *Request) *FileResponse {
+	response := NewFileResponse(*request)
+
+	if err := os.MkdirAll(request.BaseFolder, 0775); err != nil {
+		response.Error = fmt.Errorf("failed to create base folder: %v", err)
+		return response
+	}
+
+	path := "/storage/get"
+	r := s.connection.R()
+
+	requestJSON, err := json.Marshal(request)
+	if err != nil {
+		response.Error = fmt.Errorf("failed to form request to storage: %v", err)
+		return response
+	}
+
+	r.SetQueryParams(map[string]string{
+		"request": string(requestJSON),
+	})
+
+	resp, err := r.SetDoNotParseResponse(true).Execute("GET", path)
+	if err != nil {
+		response.Error = fmt.Errorf("failed to send request: %v", err)
+		return response
+	}
+	defer resp.RawBody().Close()
+
+	if resp.StatusCode() != http.StatusOK {
+		if resp.StatusCode() == http.StatusNotFound {
+			response.Error = ErrStorageFileNotFound
+		} else {
+			response.Error = fmt.Errorf("get request failed with status: %v", resp.Status())
+		}
+		return response
+	}
+
+	var filename string
+	if request.DownloadFilename != nil && *request.DownloadFilename != "" {
+		filename = *request.DownloadFilename
+	} else {
+		// Extract filename from Content-Disposition header
+		contentDisposition := resp.Header().Get("Content-Disposition")
+		if contentDisposition != "" {
+			_, params, err := mime.ParseMediaType(contentDisposition)
+			if err == nil && params["filename"] != "" {
+				filename = params["filename"]
+			}
+		}
+	}
+
+	if filename == "" {
+		response.Error = fmt.Errorf("can't extract filename from DownloadFilename or Content-Disposition header")
+		return response
+	}
+
+	filePath := filepath.Join(request.BaseFolder, filename)
+	file, err := os.Create(filePath)
+	if err != nil {
+		response.Error = fmt.Errorf("failed to create file: %v", err)
+		return response
+	}
+	defer file.Close()
+
+	written, err := io.Copy(file, resp.RawBody())
+	if err != nil {
+		response.Error = fmt.Errorf("failed to write to file: %v", err)
+		return response
+	}
+
+	response.Filename = filename
+	response.BaseFolder = request.BaseFolder
+	response.Size = uint64(written)
+	return response
 }
 
 func (s *Connector) Upload(request *Request) *Response {
-	return nil
+	response := &Response{R: *request}
+
+	if request.File == nil {
+		response.Error = fmt.Errorf("file for upload is not specified")
+		return response
+	}
+
+	path := "/storage/upload"
+	r := s.connection.R()
+
+	requestJSON, err := json.Marshal(request)
+	if err != nil {
+		response.Error = fmt.Errorf("failed to form request to storage: %v", err)
+		return response
+	}
+
+	r.SetFormData(map[string]string{
+		"request": string(requestJSON),
+	})
+
+	// request.StorageFilename can be empty
+	r.SetFileReader("file", request.StorageFilename, request.File)
+
+	resp, err := r.Post(path)
+	if err != nil {
+		response.Error = fmt.Errorf("failed to send request: %v", err)
+		return response
+	}
+
+	if resp.IsError() {
+		response.Error = fmt.Errorf("upload failed with status: %v", resp.Status())
+		return response
+	}
+
+	return response
 }
 
 func (s *Connector) Delete(request *Request) *Response {
-	return nil
+	response := &Response{R: *request}
+
+	path := "/storage/remove"
+	r := s.connection.R()
+
+	requestJSON, err := json.Marshal(request)
+	if err != nil {
+		response.Error = fmt.Errorf("failed to form request to storage: %v", err)
+		return response
+	}
+
+	r.SetFormData(map[string]string{
+		"request": string(requestJSON),
+	})
+
+	resp, err := r.Delete(path)
+	if err != nil {
+		response.Error = fmt.Errorf("failed to send request: %v", err)
+		return response
+	}
+
+	if resp.IsError() {
+		response.Error = fmt.Errorf("delete failed with status: %v", resp.Status())
+		return response
+	}
+
+	return response
 }

common/connectors/storageconn/errors.go

@@ -0,0 +1,5 @@
+package storageconn
+
+import "errors"
+
+var ErrStorageFileNotFound = errors.New("file not found in storage")

common/connectors/storageconn/structs.go

@@ -12,6 +12,12 @@ type Request struct {
 	// Should be always specified
 	Resource resource.Type `json:"resource"`
 
+	/*
+		Resource must always has exactly one of ProblemId and SubmitID
+		Including, only TestID cannot be specified
+		ID=0 is considered absent
+	*/
+
 	// If resource is part of problem, ProblemID is used
 	ProblemID uint64 `json:"problemID"`
 	// If resource is part of submit, SubmitID is used
@@ -22,46 +28,57 @@ type Request struct {
 	// For any download, BaseFolder should be specified. The files with original filenames will be placed there
 	BaseFolder string `json:"-"`
 
-	// For uploads, Files should be specified. Filename is key in map, file data should be read from value
-	Files map[string]io.Reader `json:"-"`
+	// Specify a custom filename for the downloaded file
+	DownloadFilename *string `json:"-"`
+
+	// For uploads, File should be specified
+	File io.Reader `json:"-"`
+
+	// If StorageFilename is not specified, Storage tries to get the filename automatically
+	StorageFilename string `json:"storageFilename"`
 }
 
 type Response struct {
 	R     Request
 	Error error
 }
 
-type ResponseFiles struct {
+type FileResponse struct {
 	Response
-	fileNames []string
-	Size      uint64
+	Filename   string `json:"filename"`
+	BaseFolder string `json:"basefolder"`
+	Size       uint64 `json:"size"`
 }
 
-func (r *ResponseFiles) File() (string, bool) {
-	if len(r.fileNames) == 0 {
-		return "", false
+func NewFileResponse(request Request) *FileResponse {
+	return &FileResponse{
+		Response:   Response{R: request, Error: nil},
+		Filename:   "",
+		BaseFolder: "",
+		Size:       0,
 	}
-	return filepath.Join(r.R.BaseFolder, r.fileNames[0]), true
 }
 
-func (r *ResponseFiles) Get(fileName string) (string, bool) {
-	for _, f := range r.fileNames {
-		if fileName == f {
-			return filepath.Join(r.R.BaseFolder, f), true
-		}
+func (r *FileResponse) GetFilePath() (string, bool) {
+	if r.BaseFolder == "" || r.Filename == "" {
+		return "", false
 	}
-	return "", false
+	return filepath.Join(r.BaseFolder, r.Filename), true
 }
 
-func (r *ResponseFiles) CleanUp() {
+// Removes BaseFolder with all files
+func (r *FileResponse) CleanUp() {
 	if r.Error != nil {
+		logger.Error("CleanUp was called for failed FileResponse: %v", r.Error)
 		return
 	}
-	if len(r.R.BaseFolder) == 0 {
+	if r.BaseFolder == "" {
+		logger.Error("CleanUp was called for empty BaseFolder name")
 		return
 	}
-	err := os.RemoveAll(r.R.BaseFolder)
+
+	err := os.RemoveAll(r.BaseFolder)
 	if err != nil {
-		logger.Panic("Can not remove resource folder, error: %s", err.Error())
+		logger.Error("Cannot remove resource folder %s: %s", r.BaseFolder, err.Error())
 	}
 }

common/constants/resource/datatype_string.go

@@ -1,4 +1,5 @@
 //go:generate go run golang.org/x/tools/cmd/stringer@latest -type=Type
+//go:generate go run golang.org/x/tools/cmd/stringer@latest -type=DataType
 
 package resource
 
@@ -16,4 +17,14 @@ const (
 	CheckerOutput
 	Interactor
 	// Will be increased
+	// Don't forget to add a new type to storage/filesystem/resource_info.go
+)
+
+type DataType int
+
+const (
+	UnknownDataType DataType = iota
+	Problem
+	Submission
+	// Will be increased
 )

common/constants/resource/resource_type.go

@@ -0,0 +1,27 @@
+# Устройство Storage
+
+## API-интерфейс
+
+Storage предоставляет три основных метода API:
+- `POST /storage/upload` — загрузка файла
+- `GET /storage/get` — получение файла
+- `DELETE /storage/remove` — удаление файла
+
+Все API-методы принимают следующие параметры:
+- `id` — уникальный идентификатор объекта (например, problem ID или submission ID)
+- `dataType` — тип данных (например, `problem` или `submission`)
+- `filepath` — путь к файлу или имя файла
+
+## Filesystem
+
+Filesystem представляет собой интерфейс для работы с файловой системой и имеет следующие методы:
+- `SaveFile` — сохранение загруженного файла
+- `RemoveFile` — удаление файла
+- `GetFilePath` — получение пути к файлу
+
+## StorageConnector
+
+Для взаимодействия с Storage из других сервисов используется `StorageConn`, который предоставляет следующие методы:
+- `Download` — загрузка файла из Storage
+- `Upload` — отправка файла в Storage
+- `Delete` — удаление файла из Storage

docs/storage.md

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"golang.org/x/net/html/charset"
-	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -178,9 +177,7 @@ func (s *JobPipelineState) uploadCheckerOutput() error {
 		Resource: resource.CheckerOutput,
 		SubmitID: uint64(s.job.Submission.ID),
 		TestID:   s.job.Test,
-		Files: map[string]io.Reader{
-			checkOutputFile: s.test.checkerOutputReader,
-		},
+		File:     s.test.checkerOutputReader,
 	}
 	resp := s.invoker.TS.StorageConn.Upload(checkerOutputRequest)
 	if resp.Error != nil {