-
Notifications
You must be signed in to change notification settings - Fork 3
/
client.go
160 lines (133 loc) · 3.81 KB
/
client.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
package monkeylearn
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strconv"
"time"
)
const (
classifierURL = "https://api.monkeylearn.com/v3/classifiers/%s/classify/"
extractorURL = "https://api.monkeylearn.com/v3/extractors/%s/extract/"
)
// Client holds the authentication data to connect to the MonkeyLearn
// API and is used as gateway to operate with the API
type Client struct {
http.Client
token string
RequestLimit, RequestRemaining int
}
// NewClient returns a new Client initialized with an authentication token
func NewClient(token string) *Client {
return &Client{token: token}
}
// Process does the appropriate call to the MonkeyLearn API and
// handles the response
func (c *Client) Process(endpoint string, data []byte) ([]Result, error) {
resp, err := c.Do(
c.newRequest(endpoint, data),
)
if err != nil { log.Panic(err) }
// We get rate limited. Do something
if resp.StatusCode == 429 {
return nil, fmt.Errorf("Request got ratelimited calling %s", endpoint)
}
// Not succesful? Better error out
if resp.StatusCode != 200 {
return nil, fmt.Errorf("Unsuccessful request: %s", resp.Status)
}
// Only if request is successful
c.updateLimits(resp)
res, err := deserializeResponse(resp)
if err != nil { log.Panic(err) }
return res, nil
}
func (c *Client) newRequest(url string, data []byte) *http.Request {
req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
if err != nil { log.Panic(err) }
req.Header.Add("Authorization", fmt.Sprintf("Token %s", c.token))
req.Header.Add("Content-Type", "application/json")
return req
}
func (c *Client) updateLimits(response *http.Response) {
var err error
c.RequestRemaining, err = strconv.Atoi(response.Header.Get("X-Query-Limit-Remaining"))
if err != nil { log.Panic(err) }
c. RequestLimit, err = strconv.Atoi(response.Header.Get("X-Query-Limit-Limit"))
if err != nil { log.Panic(err) }
}
// Result holds the results of processing a document be it either an
// extraction or a classification
type Result struct {
Text string
ExternalID string `json:"external_id"`
IsError bool `json:"error"`
ErrorDetail string `json:"error_detail"`
Classifications []Classification
Extractions []Extraction
}
// Error returns an error if the processing had an error
func (r Result) Error() error {
if r.IsError {
return fmt.Errorf(r.ErrorDetail)
}
return nil
}
func MargeResultList(lists ...[]Result) []Result {
dict := make(map[string]Result)
for _, list := range lists {
for _, result := range list {
index := result.ExternalID
val, ok := dict[index]
if !ok {
dict[index] = result
} else {
dict[index] = mergeResult(val, result)
}
}
}
values := make([]Result, 0, len(dict))
for _, v := range dict {
values = append(values, v)
}
return values
}
func mergeResult(a, b Result) Result {
a.Classifications = append(a.Classifications, b.Classifications...)
a.Extractions = append(a.Extractions, b.Extractions...)
return a
}
// Classification holds the classification information related to a
// processed document
type Classification struct {
TagName string `json:"tag_name"`
TagID int `json:"tag_id"`
Confidence float64
}
// Extraction represents an instance of extracted elements from a
// document
type Extraction struct {
TagName string `json:"tag_name"`
ExtractedText string `json:"extracted_text"`
OffsetSpan []int `json:"offset_span"`
ParsedValue interface{} `json:"parsed_value"`
}
func startTimer(name string) func() {
t := time.Now()
return func() {
d := time.Now().Sub(t)
log.Println(name, "took", d)
}
}
func deserializeResponse(response *http.Response) ([]Result, error) {
defer response.Body.Close()
body, err := ioutil.ReadAll(response.Body)
if err != nil { return nil, err }
var res []Result
err = json.Unmarshal(body, &res)
if err != nil { return nil, err }
return res, nil
}