forked from NothingCw/cf
/
ossls.go
333 lines (314 loc) · 9.44 KB
/
ossls.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
package alioss
import (
"fmt"
"strconv"
"github.com/AlecAivazis/survey/v2"
"github.com/fwoawr/cf/pkg/util/errutil"
"github.com/fwoawr/cf/pkg/util/pubutil"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
log "github.com/sirupsen/logrus"
"github.com/fwoawr/cf/pkg/cloud"
"github.com/fwoawr/cf/pkg/util"
"github.com/fwoawr/cf/pkg/util/cmdutil"
)
type Bucket = cloud.Resource
type Object struct {
BucketName string
ObjectNumber int
ObjectSize int64
}
type Acl struct {
BucketName string
Acl string
}
type objectContents struct {
Key string
Size int64
LastModified string
}
type error interface {
Error() string
}
var (
objectNum int
ObjectSize int64
objects []objectContents
TimestampType = util.ReturnTimestampType("alibaba", "oss")
header = []string{"序号 (SN)", "名称 (Name)", "存储桶 ACL (Bucket ACL)", "对象数量 (Object Number)", "存储桶大小 (Bucket Size)", "区域 (Region)", "存储桶地址 (Bucket URL)"}
)
func (o *OSSCollector) ListBuckets(ossLsBucket string, ossLsRegion string) ([]Bucket, error) {
var (
out []Bucket
err error
)
if ossLsBucket == "all" {
region := cloud.GetGlobalRegions()[0]
o.OSSClient(region)
var size = 1000
marker := oss.Marker("")
for {
var lbr oss.ListBucketsResult
lbr, err = o.Client.ListBuckets(oss.MaxKeys(size), marker)
marker = oss.Marker(lbr.NextMarker)
for _, bucket := range lbr.Buckets {
obj := Bucket{Name: bucket.Name,
Region: bucket.Location[4:],
}
out = append(out, obj)
}
if !lbr.IsTruncated {
break
}
}
errutil.HandleErrNoExit(err)
} else {
var bucket Bucket
if ossLsRegion == "all" {
bucket = Bucket{
Name: ossLsBucket,
Region: o.GetBucketRegion(ossLsBucket),
}
} else {
bucket = Bucket{
Name: ossLsBucket,
Region: ossLsRegion,
}
}
out = append(out, bucket)
}
return out, err
}
func (o *OSSCollector) GetBucketRegion(bucketName string) string {
var err error
region := cloud.GetGlobalRegions()[0]
o.OSSClient(region)
var lbr oss.GetBucketInfoResult
lbr, err = o.Client.GetBucketInfo(bucketName)
errutil.HandleErr(err)
region = lbr.BucketInfo.Location[4:]
return region
}
func (o *OSSCollector) ListObjects(bucketName string, ossLsObjectNumber string, ossLsRegion string) ([]Object, []objectContents) {
var (
size int
out []Object
Buckets []Bucket
)
if ossLsObjectNumber == "all" {
size = 1000
} else {
var err error
size, err = strconv.Atoi(ossLsObjectNumber)
errutil.HandleErr(err)
}
marker := oss.Marker("")
OSSCollector := &OSSCollector{}
Buckets, _ = OSSCollector.ListBuckets(bucketName, ossLsRegion)
if bucketName != "all" {
for _, obj := range Buckets {
if obj.Name == bucketName {
Buckets = nil
Buckets = append(Buckets, obj)
}
}
}
for _, j := range Buckets {
BucketName := j.Name
log.Infof("正在获取 %s 存储桶的数据 (Fetching data for %s bucket)", BucketName, BucketName)
region := j.Region
o.OSSClient(region)
bucket, err := o.Client.Bucket(BucketName)
errutil.HandleErr(err)
objects = nil
getAllObjects(bucket, marker, size, ossLsObjectNumber)
log.Debugf("在 %s 存储桶中找到了 %d 个对象 (Found %d Objects in %s Bucket)", BucketName, objectNum, objectNum, BucketName)
obj := Object{
BucketName: BucketName,
ObjectNumber: objectNum,
ObjectSize: ObjectSize,
}
objectNum = 0
ObjectSize = 0
out = append(out, obj)
}
return out, objects
}
func getAllObjects(bucket *oss.Bucket, marker oss.Option, size int, ossLsObjectNumber string) {
lor, err := bucket.ListObjects(oss.MaxKeys(size), marker)
errutil.HandleErr(err)
marker = oss.Marker(lor.NextMarker)
objectNum = objectNum + len(lor.Objects)
if objectNum%10000 == 0 && objectNum != 0 {
log.Infof("当前已获取到 %d 条数据 (%d pieces of data have been obtained)", objectNum, objectNum)
}
for _, k := range lor.Objects {
ObjectSize = ObjectSize + k.Size
obj := objectContents{
Key: k.Key,
Size: k.Size,
LastModified: k.LastModified.Format("2006-01-02 15:04:05"),
}
objects = append(objects, obj)
}
log.Tracef("Next Marker: %s", lor.NextMarker)
NextMarker := lor.NextMarker
if objectNum == 100000 {
var name bool
prompt := &survey.Confirm{
Message: "已查询到 10w 条对象,是否继续?如果继续可能会耗费较长时间。(Found up to 100,000 objects, want to continue? If you continue, it may take a long time)",
Default: true,
}
_ = survey.AskOne(prompt, &name)
if !name {
NextMarker = ""
log.Infoln("已停止继续查询对象,您还可以通过 -n 参数指定您想要查询对象的数量。(Has stopped continuing to query objects. You can specify the number of objects to query with the -n parameter.)")
}
}
if ossLsObjectNumber != "all" {
ossLsObjectNumberInt, err := strconv.Atoi(ossLsObjectNumber)
errutil.HandleErr(err)
if objectNum >= ossLsObjectNumberInt {
NextMarker = ""
objectNum = ossLsObjectNumberInt
objects = objects[0:objectNum]
}
}
if NextMarker != "" {
getAllObjects(bucket, marker, size, ossLsObjectNumber)
}
}
func (o *OSSCollector) GetBucketACL(ossLsBucket string, ossLsRegion string) []Acl {
OSSCollector := &OSSCollector{}
Buckets, _ := OSSCollector.ListBuckets(ossLsBucket, ossLsRegion)
var (
out []Acl
obj Acl
)
for _, j := range Buckets {
BucketName := j.Name
region := j.Region
o.OSSClient(region)
gbar, err := o.Client.GetBucketACL(BucketName)
//errutil.HandleErr(err)
if err != nil {
BucketACL := gbar.ACL
if BucketACL == "private" {
BucketACL = "私有 (Private)"
} else if BucketACL == "public-read" {
BucketACL = "公共读 (Public Read)"
} else if BucketACL == "public-read-write" {
BucketACL = "公共读写 (Public Read Write)"
}
obj = Acl{
BucketName: BucketName,
Acl: BucketACL,
}
} else {
obj = Acl{
BucketName: BucketName,
Acl: "N/A",
}
}
out = append(out, obj)
}
return out
}
func PrintBucketsListRealTime(region string, ossLsObjectNumber string, ossLsBucket string) {
var (
num int
dataLen int
Buckets []Bucket
Objects []Object
ACL []Acl
)
OSSCollector := &OSSCollector{}
Buckets, _ = OSSCollector.ListBuckets(ossLsBucket, region)
log.Infof("获取到 %d 条 OSS Bucket 信息 (Obtained %d pieces of OSS Bucket information)", len(Buckets), len(Buckets))
if ossLsBucket == "all" {
ACL = OSSCollector.GetBucketACL("all", region)
Objects, _ = OSSCollector.ListObjects("all", ossLsObjectNumber, region)
} else {
ACL = OSSCollector.GetBucketACL(ossLsBucket, region)
Objects, _ = OSSCollector.ListObjects(ossLsBucket, ossLsObjectNumber, region)
}
for _, o := range Buckets {
if region == "all" {
num = len(Buckets)
} else {
if region == o.Region {
num = num + 1
}
}
}
var data = make([][]string, num)
num = 0
for i, o := range Buckets {
if region == "all" {
SN := strconv.Itoa(i + 1)
ObjectNumber := strconv.Itoa(Objects[i].ObjectNumber)
ObjectSize := pubutil.FormatFileSize(Objects[i].ObjectSize)
BucketACL := ACL[i].Acl
BucketURL := fmt.Sprintf("https://%s.oss-%s.aliyuncs.com", o.Name, o.Region)
data[i] = []string{SN, o.Name, BucketACL, ObjectNumber, ObjectSize, o.Region, BucketURL}
dataLen = dataLen + 1
} else {
if region == o.Region {
ObjectNumber := strconv.Itoa(Objects[i].ObjectNumber)
ObjectSize := pubutil.FormatFileSize(Objects[i].ObjectSize)
BucketACL := ACL[i].Acl
BucketURL := fmt.Sprintf("https://%s.oss-%s.aliyuncs.com", o.Name, o.Region)
data[num] = []string{strconv.Itoa(num + 1), o.Name, BucketACL, ObjectNumber, ObjectSize, o.Region, BucketURL}
num = num + 1
dataLen = dataLen + 1
}
}
}
var td = cloud.TableData{Header: header, Body: data}
if dataLen == 0 && ossLsBucket != "all" {
log.Info("没发现存储桶 (No Buckets Found)")
} else if dataLen == 0 && ossLsBucket == "all" {
log.Info("没发现存储桶,注意有些账号需要指定存储桶名称才能列出,可以使用 -b 命令指定存储桶名称 (No bucket found, note that some accounts require a bucket name to be listed, you can specify the bucket name with the -b command.)")
} else {
Caption := "OSS 资源 (OSS resources)"
cloud.PrintTable(td, Caption)
util.WriteTimestamp(TimestampType)
}
cmdutil.WriteCacheFile(td, "alibaba", "oss", region, "all")
}
func PrintBucketsListHistory(region string, ossLsBucket string) {
cmdutil.PrintOSSCacheFile(header, region, "alibaba", "OSS", ossLsBucket)
}
func PrintBucketsList(region string, lsFlushCache bool, ossLsObjectNumber string, ossLsBucket string) {
if lsFlushCache {
PrintBucketsListRealTime(region, ossLsObjectNumber, ossLsBucket)
} else {
oldTimestamp := util.ReadTimestamp(TimestampType)
if oldTimestamp == 0 {
PrintBucketsListRealTime(region, ossLsObjectNumber, ossLsBucket)
} else if util.IsFlushCache(oldTimestamp) {
PrintBucketsListRealTime(region, ossLsObjectNumber, ossLsBucket)
} else {
util.TimeDifference(oldTimestamp)
PrintBucketsListHistory(region, ossLsBucket)
}
}
}
func ReturnBucketList(ossLsBucket string, ossLsRegion string) []string {
var (
buckets []string
ossCache []pubutil.OSSCache
)
OSSCollector := &OSSCollector{}
ossCache = cmdutil.ReadOSSCache("alibaba")
if len(ossCache) == 0 {
BucketsList, _ := OSSCollector.ListBuckets(ossLsBucket, ossLsRegion)
for _, v := range BucketsList {
buckets = append(buckets, v.Name)
}
} else {
for _, v := range ossCache {
buckets = append(buckets, v.Name)
}
}
return buckets
}