-
Notifications
You must be signed in to change notification settings - Fork 0
/
minio.go
113 lines (101 loc) · 3.12 KB
/
minio.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
package greynetanalysis
import (
"context"
"fmt"
"io"
"log"
"os"
"strings"
"time"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
//GreynetMinio defines minio object
type GreynetMinio struct {
client *minio.Client
endpoint, bucket, accessid, accesskey string
Ctx context.Context
}
const STARDUSTBUCKETNAME = "imc-paper-stardust"
const ENDPOINTVAR = "MIO_ENDPOINT"
const ACCESSKEYVAR = "MIO_ACCESS_KEY_ID"
const SECRETKEYVAR = "MIO_SECRET_KEY"
//NewGreynetMinioiwthContext initials minio object
func NewGreynetMinioiwthContext(ctx context.Context) GreynetMinio {
var err error
g := GreynetMinio{Ctx: ctx}
//read from environment variables
g.endpoint = os.Getenv(ENDPOINTVAR)
g.accessid = os.Getenv(ACCESSKEYVAR)
g.accesskey = os.Getenv(SECRETKEYVAR)
g.bucket = STARDUSTBUCKETNAME
log.Println(g)
g.client, err = minio.New(g.endpoint, &minio.Options{
Creds: credentials.NewStaticV4(g.accessid, g.accesskey, ""),
Secure: true,
})
if err != nil {
log.Fatal("failed to init minio", err)
}
return g
}
func (gmio GreynetMinio) ListNTpcapsbyDate(d time.Time) []string {
ctx, cancel := context.WithCancel(gmio.Ctx)
defer cancel()
objprefix := fmt.Sprintf("datasource=ucsd-nt/year=%d/month=%02d/day=%02d", d.Year(), d.Month(), d.Day())
objchan := gmio.client.ListObjects(ctx, gmio.bucket, minio.ListObjectsOptions{
Prefix: objprefix,
Recursive: true,
})
objs := make([]string, 0)
for object := range objchan {
if object.Err != nil {
log.Println("list object error", object.Err)
} else {
//only list pcap files
if strings.Contains(object.Key, ".pcap") {
objs = append(objs, object.Key)
}
}
}
return objs
}
func (gmio GreynetMinio) ListNTannotationbyDate(d time.Time)[]string{
ctx, cancel := context.WithCancel(gmio.Ctx)
defer cancel()
objprefix := fmt.Sprintf("datasource=ucsd-nt/year=%d/month=%02d/day=%02d", d.Year(), d.Month(), d.Day())
objchan := gmio.client.ListObjects(ctx, gmio.bucket, minio.ListObjectsOptions{
Prefix: objprefix,
Recursive: true,
})
objs := make([]string, 0)
for object := range objchan {
if object.Err != nil {
log.Println("list object error", object.Err)
} else {
//only list pcap files
if strings.Contains(object.Key, ".jsonl") {
objs = append(objs, object.Key)
}
}
}
return objs
}
func (gmio GreynetMinio) GetObject(objpath string) *minio.Object {
object, err := gmio.client.GetObject(gmio.Ctx, gmio.bucket, objpath, minio.GetObjectOptions{})
if err != nil {
log.Println(err)
return nil
}
return object
}
func (gmio GreynetMinio) PutMetaData(objpath string, reader io.Reader) minio.UploadInfo {
log.Println("putmetadata starts", objpath)
uploadinfo, err := gmio.client.PutObject(gmio.Ctx, gmio.bucket, objpath, reader, -1, minio.PutObjectOptions{ContentType: "application/json", ContentEncoding: "gzip"})
//uploadinfo, err := gmio.client.PutObject(gmio.Ctx, gmio.bucket, objpath, reader, -1, minio.PutObjectOptions{})
if err != nil {
log.Println("upload", objpath, "error", err)
}
//log.Println("uploaded", uploadinfo)
return uploadinfo
}