Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix bugs to xskyapi #582

Merged
merged 1 commit into from Jul 31, 2023
Merged

fix bugs to xskyapi #582

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
97 changes: 58 additions & 39 deletions inputs/xskyapi/xskyapi.go
View file
Open in desktop
Expand Up @@ -155,26 +155,29 @@ func (ins *Instance) gather(slist *types.SampleList, server string, token string
if config.Config.DebugMode {
log.Println("D! xskyapi... server:", server)
}

pageSize := 500
offset := 0
labels := map[string]string{"server": server}
fields := make(map[string]interface{})

var currentUrl string

// acquire quota data of 3 mainstream distributed storage service provided by Xsky
switch ins.DssType {
case "oss": // object storage

// oss users
osUsers := OsUsers{}
urlUsers := server + "/v1/os-users"
currentUrl = server + "/v1/os-users"
for {
resp, _, err := ins.sendRequest(urlUsers, token, offset)
resp, _, err := ins.sendRequest(currentUrl, token, offset, pageSize)
if err != nil {
log.Println("E! failed to send request to xskyapi url:", urlUsers, "error:", err)
log.Println("E! failed to send request to xskyapi url:", currentUrl, "error:", err)
}

er := json.Unmarshal(resp, &osUsers)
if er != nil {
fmt.Printf("Parsing JSON string exception：%s\n", err)
err = json.Unmarshal(resp, &osUsers)
if err != nil {
fmt.Printf("E! Parsing JSON string exception：%s\n", err)
}

for _, user := range osUsers.OsUser {
Expand All @@ -184,20 +187,25 @@ func (ins *Instance) gather(slist *types.SampleList, server string, token string
fields["oss_user_used_size"] = user.Samples[0].AllocatedSize
slist.PushSamples(inputName, fields, labels)
}
if osUsers.Page.Limit > osUsers.Page.Count {

offset = offset + pageSize
if offset >= osUsers.Page.TotalCount {
break
}
offset = osUsers.Page.Offset + osUsers.Page.Limit
}

// oss buckets
labels = map[string]string{"server": server}
fields = make(map[string]interface{})
offset = 0

osBuckets := OsBuckets{}
urlBuckets := server + "/v1/os-buckets"
currentUrl = server + "/v1/os-buckets"

for {
resp, _, err := ins.sendRequest(urlBuckets, token, offset)
resp, _, err := ins.sendRequest(currentUrl, token, offset, pageSize)
if err != nil {
log.Println("E! failed to send request to xskyapi url:", urlBuckets, "error:", err)
log.Println("E! failed to send request to xskyapi url:", currentUrl, "error:", err)
}
err = json.Unmarshal(resp, &osBuckets)
if err != nil {
Expand All @@ -213,54 +221,58 @@ func (ins *Instance) gather(slist *types.SampleList, server string, token string
fields["oss_bucket_used_size"] = bucket.Samples[0].AllocatedSize
slist.PushSamples(inputName, fields, labels)
}
if osBuckets.Page.Limit > osBuckets.Page.Count {

offset = offset + pageSize
if offset >= osBuckets.Page.TotalCount {
break
}
offset = osBuckets.Page.Offset + osBuckets.Page.Limit
}

case "gfs":

// gfs dfs

dfsQuotas := DfsQuotas{}
urlDfs := server + "/v1/dfs-quotas"
currentUrl = server + "/v1/dfs-quotas"

for {
resp, _, err := ins.sendRequest(urlDfs, token, offset)
resp, _, err := ins.sendRequest(currentUrl, token, offset, pageSize)
if err != nil {
log.Println("E! failed to send request to xskyapi url:", urlDfs, "error:", err)
log.Println("E! failed to send request to xskyapi url:", currentUrl, "error:", err)
}
er := json.Unmarshal(resp, &dfsQuotas)
if er != nil {
fmt.Printf("Parsing JSON string exception：%s\n", err)
}

labels := map[string]string{"server": server}

for _, dfsQuota := range dfsQuotas.DfsQuota {
labels["name"] = dfsQuota.DfsPath.Name
labels["id"] = strconv.Itoa(dfsQuota.DfsPath.ID)
fields["dfs_quota"] = dfsQuota.SizeHardQuota
slist.PushSamples(inputName, fields, labels)
}
if dfsQuotas.Page.Limit > dfsQuotas.Page.Count {

offset = offset + pageSize
if offset >= dfsQuotas.Page.TotalCount {
break
}
offset = dfsQuotas.Page.Offset + dfsQuotas.Page.Limit
}

// gfs block volumes
labels = map[string]string{"server": server}
fields = make(map[string]interface{})

offset = 0

blockVolumes := BlockVolumes{}
urlBV := server + "/v1/block-volumes"
currentUrl = server + "/v1/block-volumes"

for {
resp, _, err := ins.sendRequest(urlBV, token, offset)
resp, _, err := ins.sendRequest(currentUrl, token, offset, pageSize)
if err != nil {
log.Println("E! failed to send request to xskyapi url:", urlDfs, "error:", err)
log.Println("E! failed to send request to xskyapi url:", currentUrl, "error:", err)
}
er := json.Unmarshal(resp, &dfsQuotas)
er := json.Unmarshal(resp, &blockVolumes)
if er != nil {
fmt.Printf("Parsing JSON string exception：%s\n", err)
}
Expand All @@ -272,23 +284,24 @@ func (ins *Instance) gather(slist *types.SampleList, server string, token string
fields["block_volume_used_size"] = blockVolume.AllocatedSize
slist.PushSamples(inputName, fields, labels)
}
if blockVolumes.Page.Limit > blockVolumes.Page.Count {

offset = offset + pageSize
if offset >= blockVolumes.Page.TotalCount {
break
}
offset = blockVolumes.Page.Offset + blockVolumes.Page.Limit
}

case "eus":

// eus-folder

fsFolders := FsFolders{}
urlDfs := server + "/v1/fs-folders"
currentUrl = server + "/v1/fs-folders"

for {
resp, _, err := ins.sendRequest(urlDfs, token, offset)
resp, _, err := ins.sendRequest(currentUrl, token, offset, pageSize)
if err != nil {
log.Println("E! failed to send request to xskyapi url:", urlDfs, "error:", err)
log.Println("E! failed to send request to xskyapi url:", currentUrl, "error:", err)
}

er := json.Unmarshal(resp, &fsFolders)
Expand All @@ -302,21 +315,24 @@ func (ins *Instance) gather(slist *types.SampleList, server string, token string
fields["dfs_quota"] = fsFolder.Size
slist.PushSamples(inputName, fields, labels)
}
if fsFolders.Page.Limit > fsFolders.Page.Count {
offset = offset + pageSize
if offset >= fsFolders.Page.TotalCount {
break
}
offset = fsFolders.Page.Offset + fsFolders.Page.Limit
}

// eus block volumes
labels = map[string]string{"server": server}
fields = make(map[string]interface{})
offset = 0

blockVolumes := BlockVolumes{}
urlBV := server + "/v1/block-volumes"
currentUrl = server + "/v1/block-volumes"

for {
resp, _, err := ins.sendRequest(urlBV, token, offset)
resp, _, err := ins.sendRequest(currentUrl, token, offset, pageSize)
if err != nil {
log.Println("E! failed to send request to xskyapi url:", urlDfs, "error:", err)
log.Println("E! failed to send request to xskyapi url:", currentUrl, "error:", err)
}

err = json.Unmarshal(resp, &blockVolumes)
Expand All @@ -333,26 +349,29 @@ func (ins *Instance) gather(slist *types.SampleList, server string, token string
fields["block_volume_used_size"] = blockVolume.AllocatedSize
slist.PushSamples(inputName, fields, labels)
}
if blockVolumes.Page.Limit > blockVolumes.Page.Count {
offset = offset + pageSize
if offset >= blockVolumes.Page.TotalCount {
break
}
offset = blockVolumes.Page.Offset + blockVolumes.Page.Limit
}
default:
log.Printf("E! dss_type %s not suppported, expected oss, gfs or eus", ins.DssType)
}
}

func (ins *Instance) sendRequest(serverURL string, token string, offset int) ([]byte, float64, error) {
func (ins *Instance) sendRequest(serverURL string, token string, offset int, pageSize int) ([]byte, float64, error) {
// Prepare URL
requestURL, _ := url.Parse(serverURL)
log.Println("D! now parseurl:", requestURL)
if config.Config.DebugMode {
log.Println("D! now parseurl:", requestURL)
}

// Prepare request query and body
data := url.Values{}

params := requestURL.Query()
params.Add("offset", strconv.Itoa(offset))
params.Add("limit", strconv.Itoa(pageSize))

requestURL.RawQuery = params.Encode()

Expand Down