-
Notifications
You must be signed in to change notification settings - Fork 78
/
stats.go
336 lines (304 loc) · 8.72 KB
/
stats.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
package stats
import (
"context"
"errors"
"github.com/bittorrent/go-btfs/utils"
"sort"
"strconv"
"strings"
"time"
"github.com/bittorrent/go-btfs/core"
"github.com/bittorrent/go-btfs/core/commands/cmdenv"
"github.com/bittorrent/go-btfs/core/commands/storage/helper"
"github.com/bittorrent/go-btfs/core/commands/storage/upload/sessions"
"github.com/bittorrent/go-btfs/core/corerepo"
"github.com/bittorrent/go-btfs/core/hub"
cmds "github.com/bittorrent/go-btfs-cmds"
nodepb "github.com/bittorrent/go-btfs-common/protos/node"
config "github.com/bittorrent/go-btfs-config"
"github.com/bittorrent/protobuf/proto"
ds "github.com/ipfs/go-datastore"
"github.com/shirou/gopsutil/v3/disk"
)
const (
localInfoOnlyOptionName = "local-only"
versionOptionName = "version"
)
// Storage Stats
//
// Includes sub-commands: info, sync
var StorageStatsCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Get node storage stats.",
ShortDescription: `
This command get node storage stats in the network.`,
},
Subcommands: map[string]*cmds.Command{
"sync": storageStatsSyncCmd,
"info": storageStatsInfoCmd,
"list": storageStatsListCmd,
},
}
// sub-commands: btfs storage stats sync
var storageStatsSyncCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Synchronize node stats.",
ShortDescription: `
This command synchronize node stats from network(hub) to local node data store.`,
},
Arguments: []cmds.Argument{},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
err := utils.CheckSimpleMode(env)
if err != nil {
return err
}
cfg, err := cmdenv.GetConfig(env)
if err != nil {
return err
}
n, err := cmdenv.GetNode(env)
if err != nil {
return err
}
return SyncStats(req.Context, cfg, n, env, true)
},
}
func SyncStats(ctx context.Context, cfg *config.Config, node *core.IpfsNode, env cmds.Environment, v2 bool) error {
sr, err := hub.QueryStats(ctx, node, v2)
if err != nil {
return err
}
stat, err := corerepo.RepoStat(ctx, node)
if err != nil {
return err
}
cfgRoot, err := cmdenv.GetConfigRoot(env)
if err != nil {
return err
}
du, err := disk.UsageWithContext(ctx, cfgRoot)
if err != nil {
return err
}
hs := &nodepb.StorageStat_Host{
Online: cfg.Experimental.StorageHostEnabled,
StorageUsed: int64(stat.RepoSize),
StorageCap: int64(stat.StorageMax),
StorageDiskTotal: int64(du.Total),
StorageDiskAvailable: int64(du.Free),
}
hs.StorageStat_HostStats = sr.StorageStat_HostStats
return SaveHostStatsIntoDatastore(ctx, node, node.Identity.Pretty(), hs)
}
func GetNowStats(ctx context.Context, cfg *config.Config, node *core.IpfsNode, env cmds.Environment, V2 bool) (hs *nodepb.StorageStat_Host, err error) {
sr, err := hub.QueryStats(ctx, node, V2)
if err != nil {
return nil, err
}
stat, err := corerepo.RepoStat(ctx, node)
if err != nil {
return nil, err
}
cfgRoot, err := cmdenv.GetConfigRoot(env)
if err != nil {
return nil, err
}
du, err := disk.UsageWithContext(ctx, cfgRoot)
if err != nil {
return nil, err
}
hs = &nodepb.StorageStat_Host{
Online: cfg.Experimental.StorageHostEnabled,
StorageUsed: int64(stat.RepoSize),
StorageCap: int64(stat.StorageMax),
StorageDiskTotal: int64(du.Total),
StorageDiskAvailable: int64(du.Free),
}
hs.StorageStat_HostStats = sr.StorageStat_HostStats
return hs, nil
}
// sub-commands: btfs storage stats info
var storageStatsInfoCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "Get node stats.",
ShortDescription: `
This command get node stats in the network from the local node data store.`,
},
Arguments: []cmds.Argument{},
Options: []cmds.Option{
cmds.BoolOption(localInfoOnlyOptionName, "l", "Return only the locally available disk stats without querying/returning the network stats.").WithDefault(false),
cmds.IntOption(versionOptionName, "v", "Get new hub score level.").WithDefault(2),
},
RunTimeout: 30 * time.Second,
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
err := utils.CheckSimpleMode(env)
if err != nil {
return err
}
cfg, err := cmdenv.GetConfig(env)
if err != nil {
return err
}
n, err := cmdenv.GetNode(env)
if err != nil {
return err
}
var v2Flag bool
v, _ := req.Options[versionOptionName].(int)
if v == 1 {
v2Flag = false
} else if v == 2 {
v2Flag = true
} else {
return errors.New("version should be 1 or 2, not other. ")
}
var hs *nodepb.StorageStat_Host
hs, err = GetNowStats(req.Context, cfg, n, env, v2Flag)
if err != nil {
return err
}
// Refresh latest repo stats
stat, err := corerepo.RepoStat(req.Context, n)
if err != nil {
return err
}
cfgRoot, err := cmdenv.GetConfigRoot(env)
if err != nil {
return err
}
du, err := disk.UsageWithContext(req.Context, cfgRoot)
if err != nil {
return err
}
hs.Online = cfg.Experimental.StorageHostEnabled
hs.StorageUsed = int64(stat.RepoSize)
hs.StorageCap = int64(stat.StorageMax)
hs.StorageDiskTotal = int64(du.Total)
hs.StorageDiskAvailable = int64(du.Free)
// Only host stats for now
return cmds.EmitOnce(res, &nodepb.StorageStat{
HostStats: *hs,
})
},
Type: nodepb.StorageStat{},
}
// sub-commands: btfs storage stats list
var storageStatsListCmd = &cmds.Command{
Helptext: cmds.HelpText{
Tagline: "List node stats.",
ShortDescription: `
This command list node stats in the network from the local node data store.`,
},
Arguments: []cmds.Argument{
cmds.StringArg("from", true, false, "list host local stats range from"),
cmds.StringArg("to", true, false, "list host local stats range to"),
},
RunTimeout: 30 * time.Second,
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
err := utils.CheckSimpleMode(env)
if err != nil {
return err
}
n, err := cmdenv.GetNode(env)
if err != nil {
return err
}
from, err := strconv.ParseInt(req.Arguments[0], 10, 64)
if err != nil {
return err
}
to, err := strconv.ParseInt(req.Arguments[1], 10, 64)
if err != nil {
return err
}
list, err := ListHostStatsFromDatastore(req.Context, n, n.Identity.String(), from, to)
if err != nil {
return err
}
// Only host stats for now
return cmds.EmitOnce(res, list)
},
Type: []*Stat_HostWithTimeStamp{},
}
const (
HostStatStorePrefix = "/host_stats/" // from btfs-hub
)
// GetHostStatsFromDatastore retrieves host storage stats based on node id
func GetHostStatsFromDatastore(ctx context.Context, node *core.IpfsNode, nodeId string) (*nodepb.StorageStat_Host, error) {
rds := node.Repo.Datastore()
qr, err := rds.Get(ctx, GetHostStatStorageKey(nodeId))
if err != nil {
return nil, err
}
var hs nodepb.StorageStat_Host
err = proto.Unmarshal(qr, &hs)
if err != nil {
return nil, err
}
return &hs, nil
}
type Stat_HostWithTimeStamp struct {
Stat nodepb.StorageStat_Host `json:"stat"`
Timestamp int64 `json:"timestamp"`
}
// ListHostStatsFromDatastore retrieves host storage stats based on node id
func ListHostStatsFromDatastore(ctx context.Context, node *core.IpfsNode, nodeId string, from int64, to int64) ([]*Stat_HostWithTimeStamp, error) {
rds := node.Repo.Datastore()
keys, err := sessions.ListKeys(rds, HostStatStorePrefix+nodeId+"/", "")
sort.Strings(keys)
if err != nil {
return nil, err
}
hosts := make([]*Stat_HostWithTimeStamp, 0)
ly, lm, ld := -1, "", -1
for _, k := range keys {
qr, err := rds.Get(ctx, ds.NewKey(k))
if err != nil {
continue
}
var hs nodepb.StorageStat_Host
err = proto.Unmarshal(qr, &hs)
if err != nil {
continue
}
split := strings.Split(k, "/")
t, err := strconv.ParseInt(split[len(split)-1], 10, 64)
if err != nil || t < from || t > to {
continue
}
year, month, day := time.Unix(t, 0).Date()
if ly == year && lm == month.String() && ld == day {
continue
}
ly, lm, ld = year, month.String(), day
hosts = append(hosts, &Stat_HostWithTimeStamp{
Stat: hs,
Timestamp: t,
})
}
return hosts, nil
}
func GetHostStatStorageKey(pid string) ds.Key {
return helper.NewKeyHelper(HostStatStorePrefix, pid)
}
func GetHostStatStorageKeyWithTimestamp(pid string) ds.Key {
return helper.NewKeyHelper(HostStatStorePrefix, pid, "/", strconv.FormatInt(time.Now().Unix(), 10))
}
// SaveHostStatsIntoDatastore overwrites host storage stats based on node id
func SaveHostStatsIntoDatastore(ctx context.Context, node *core.IpfsNode, nodeId string,
stats *nodepb.StorageStat_Host) error {
rds := node.Repo.Datastore()
b, err := proto.Marshal(stats)
if err != nil {
return err
}
err = rds.Put(ctx, GetHostStatStorageKey(nodeId), b)
if err != nil {
return err
}
err = rds.Put(ctx, GetHostStatStorageKeyWithTimestamp(nodeId), b)
if err != nil {
return err
}
return nil
}