Skip to content

Commit 5bff50e

Browse files
author
Artem Titov
committed
Add storage
1 parent 22fc466 commit 5bff50e

File tree

10 files changed

+231
-12
lines changed

10 files changed

+231
-12
lines changed

‎common/config/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ type Config struct {
1515
LogLevel *int `yaml:"LogLevel,omitempty"`
1616

1717
Invoker *InvokerConfig `yaml:"Invoker,omitempty"`
18+
Storage *StorageConfig `yaml:"Invoker,omitempty"`
1819
// TODO: Add instances here
1920

2021
DB DBConfig `yaml:"DB"`

‎common/config/storage.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package config
2+
3+
type StorageConfig struct {
4+
StoragePath string `yaml:"StoragePath"`
5+
}
6+
7+
func FillInStorageConfig(config *StorageConfig) {
8+
if len(config.StoragePath) == 0 {
9+
panic("No storage path specified")
10+
}
11+
}

‎go.mod‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@ toolchain go1.22rc1
66

77
require (
88
github.com/gin-gonic/gin v1.10.0
9-
github.com/go-resty/resty/v2 v2.16.2
109
github.com/lib/pq v1.10.9
11-
github.com/otiai10/copy v1.14.1
1210
github.com/xorcare/pointer v1.2.2
1311
gopkg.in/yaml.v3 v3.0.1
1412
gorm.io/driver/postgres v1.5.9

‎go.sum‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
2424
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
2525
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
2626
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
27-
github.com/go-resty/resty/v2 v2.16.2 h1:CpRqTjIzq/rweXUt9+GxzzQdlkqMdt8Lm/fuK/CAbAg=
28-
github.com/go-resty/resty/v2 v2.16.2/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU=
2927
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
3028
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
3129
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
@@ -49,11 +47,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
4947
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
5048
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
5149
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
52-
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
5350
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
5451
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
55-
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
56-
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
5752
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
5853
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
5954
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
@@ -67,7 +62,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
6762
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
6863
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
6964
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
70-
github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I=
7165
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
7266
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
7367
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -107,17 +101,13 @@ golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
107101
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
108102
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
109103
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
110-
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
111-
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
112104
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
113105
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
114106
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
115107
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
116108
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
117-
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
118109
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
119110
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
120-
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
121111
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
122112
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
123113
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎main.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,15 @@ import (
44
"os"
55
"testing_system/common"
66
"testing_system/invoker"
7+
"testing_system/storage"
78
)
89

910
func main() {
1011
configPath := os.Args[1]
1112
ts := common.InitTestingSystem(configPath)
1213
if ts.Config.Invoker != nil {
1314
invoker.SetupInvoker(ts)
15+
storage.SetupStorage(ts)
1416
}
1517
ts.Run()
1618
}

‎storage/filesystem/filesystem.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package filesystem
2+
3+
import (
4+
"fmt"
5+
"mime/multipart"
6+
"os"
7+
"path/filepath"
8+
9+
"github.com/gin-gonic/gin"
10+
)
11+
12+
type Filesystem struct {
13+
Basepath string
14+
BlockSize int
15+
}
16+
17+
func CreateFilesystem(basepath string) IFilesystem {
18+
return &Filesystem{Basepath: basepath, BlockSize: 3}
19+
}
20+
21+
func (filesystem *Filesystem) generatePathFromID(prefix string, id string) string {
22+
var parts []string
23+
parts = append(parts, prefix)
24+
for i := 0; i < len(id); i += filesystem.BlockSize {
25+
end := min(i+filesystem.BlockSize, len(id))
26+
parts = append(parts, id[i:end])
27+
}
28+
return filepath.Join(parts...)
29+
}
30+
31+
func (filesystem *Filesystem) SaveFile(prefix string, id string, filename string, file *multipart.FileHeader, c *gin.Context) error {
32+
subpath := filesystem.generatePathFromID(prefix, id)
33+
fmt.Println(filesystem.Basepath)
34+
fullFilename := filepath.Join(filesystem.Basepath, subpath, filename)
35+
return c.SaveUploadedFile(file, fullFilename)
36+
}
37+
38+
func (filesystem *Filesystem) RemoveFile(prefix string, id string, filename string) error {
39+
subpath := filesystem.generatePathFromID(prefix, id)
40+
fullFilename := filepath.Join(filesystem.Basepath, subpath, filename)
41+
42+
if err := os.Remove(fullFilename); err != nil {
43+
return fmt.Errorf("failed to remove file: %w", err)
44+
}
45+
46+
dir := filepath.Dir(fullFilename)
47+
for dir != filesystem.Basepath && dir != "." {
48+
if err := os.Remove(dir); err != nil {
49+
break
50+
}
51+
dir = filepath.Dir(dir)
52+
}
53+
54+
return nil
55+
}
56+
57+
func (filesystem *Filesystem) GetFilePath(prefix string, id string, filename string) (string, error) {
58+
subpath := filesystem.generatePathFromID(prefix, id)
59+
fullFilename := filepath.Join(filesystem.Basepath, subpath, filename)
60+
61+
if _, err := os.Stat(fullFilename); os.IsNotExist(err) {
62+
return "", fmt.Errorf("file not found")
63+
}
64+
return fullFilename, nil
65+
}

‎storage/filesystem/interface.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package filesystem
2+
3+
import (
4+
"mime/multipart"
5+
6+
"github.com/gin-gonic/gin"
7+
)
8+
9+
type IFilesystem interface {
10+
SaveFile(prefix string, id string, filename string, file *multipart.FileHeader, c *gin.Context) error
11+
RemoveFile(prefix string, id string, filename string) error
12+
GetFilePath(prefix string, id string, filename string) (string, error)
13+
}

‎storage/handlers.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package storage
2+
3+
import (
4+
"net/http"
5+
"testing_system/lib/logger"
6+
7+
"github.com/gin-gonic/gin"
8+
)
9+
10+
func (s *Storage) HandleUpload(c *gin.Context) {
11+
id, dataType, filepath, err := getInfo(c)
12+
13+
if err != nil {
14+
return
15+
}
16+
17+
file, err := c.FormFile("file")
18+
19+
if err != nil {
20+
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to read file"})
21+
return
22+
}
23+
24+
err = s.filesystem.SaveFile(dataType, id, filepath, file, c)
25+
26+
if err != nil {
27+
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to save file"})
28+
logger.Error("Failed to save file: id=%s, dataType=%s, filePath=%s\n %v", id, dataType, filepath, err)
29+
return
30+
}
31+
32+
c.JSON(http.StatusOK, gin.H{"message": "File uploaded successfully"})
33+
}
34+
35+
func (s *Storage) HandleRemove(c *gin.Context) {
36+
id, dataType, filepath, err := getInfo(c)
37+
38+
if err != nil {
39+
return
40+
}
41+
42+
err = s.filesystem.RemoveFile(dataType, id, filepath)
43+
44+
if err != nil {
45+
c.JSON(http.StatusNotFound, gin.H{"error": "Failed to remove file"})
46+
logger.Error("Failed to save file: id=%s, dataType=%s, filePath=%s\n %v", id, dataType, filepath, err)
47+
return
48+
}
49+
50+
c.JSON(http.StatusOK, gin.H{"message": "File removed successfully"})
51+
}
52+
53+
func (s *Storage) HandleGet(c *gin.Context) {
54+
id, dataType, filepath, err := getInfo(c)
55+
56+
if err != nil {
57+
return
58+
}
59+
60+
filepath, err = s.filesystem.GetFilePath(dataType, id, filepath)
61+
62+
if err != nil {
63+
c.JSON(http.StatusNotFound, gin.H{"error": "Failed to get file"})
64+
return
65+
}
66+
67+
c.File(filepath)
68+
}

‎storage/helpers.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package storage
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
7+
"github.com/gin-gonic/gin"
8+
)
9+
10+
var validDataTypes = map[string]struct{}{
11+
"submission": {},
12+
"problem": {},
13+
}
14+
15+
func isValidDataType(dataType string) bool {
16+
_, exists := validDataTypes[dataType]
17+
return exists
18+
}
19+
20+
func getInfo(c *gin.Context) (string, string, string, error) {
21+
id := c.Query("id")
22+
dataType := c.Query("dataType")
23+
filepath := c.Query("filepath")
24+
25+
if id == "" {
26+
c.JSON(http.StatusBadRequest, gin.H{"error": "Missing id"})
27+
return "", "", "", fmt.Errorf("missing id")
28+
}
29+
30+
if filepath == "" {
31+
c.JSON(http.StatusBadRequest, gin.H{"error": "Missing filepath"})
32+
return "", "", "", fmt.Errorf("missing id")
33+
}
34+
35+
if !isValidDataType(dataType) {
36+
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid dataType"})
37+
return "", "", "", fmt.Errorf("invalid dataType")
38+
}
39+
40+
return id, dataType, filepath, nil
41+
}

‎storage/storage.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package storage
2+
3+
import (
4+
"testing_system/common"
5+
"testing_system/common/config"
6+
"testing_system/lib/logger"
7+
"testing_system/storage/filesystem"
8+
)
9+
10+
type Storage struct {
11+
TS *common.TestingSystem
12+
13+
filesystem filesystem.IFilesystem
14+
}
15+
16+
func SetupStorage(ts *common.TestingSystem) {
17+
if ts.Config.Storage == nil {
18+
logger.Info("Storage is not configured, skipping storage start")
19+
return
20+
}
21+
config.FillInStorageConfig(ts.Config.Storage)
22+
23+
r := ts.Router.Group("/storage/")
24+
25+
storage := &Storage{TS: ts, filesystem: filesystem.CreateFilesystem(ts.Config.Storage.StoragePath)}
26+
27+
r.POST("/upload", storage.HandleUpload)
28+
r.DELETE("/remove", storage.HandleRemove)
29+
r.GET("/get", storage.HandleGet)
30+
}

0 commit comments

Comments
 (0)