/
httpcache.go
132 lines (121 loc) · 3.59 KB
/
httpcache.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
// Copyright 2018 Fedir RYKHTIK. All rights reserved.
// Use of this source code is governed by the GNU GPL 3.0
// license that can be found in the LICENSE file.
// Package httpcache provides an interface to HTTP request static cache,
//
// Usage example :
// func main() {
// url := "https://api.github.com/repos/astaxie/beego/contributors"
// body := httpcache.MakeCachedHTTPRequest(url)
// jsonResp, linkHeader, _ := httpcache.ReadResp(body)
// fmt.Printf("%s\n%s", jsonResp, linkHeader)
// }
//
package httpcache
import (
"bufio"
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httputil"
"os"
"time"
)
var cacheTTL = 3600
var httpClient = &http.Client{Timeout: 10 * time.Second}
const dumpBody = true
// GetFilename gets encoded filename for cache usage
func GetFilename(url string) string {
encoder := sha256.New()
encoder.Write([]byte(url))
return hex.EncodeToString(encoder.Sum(nil))
}
// MakeHTTPRequest makes a non-cacheable request to the external URL
func MakeHTTPRequest(url string) ([]byte, int, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatal("Cannont prepare the HTTP request", err)
}
if os.Getenv("GH_USR") != "" && os.Getenv("GH_PASS") != "" {
req.SetBasicAuth(os.Getenv("GH_USR"), os.Getenv("GH_PASS"))
}
resp, err := httpClient.Do(req)
if err != nil {
log.Fatal("Cannot process the HTTP request", err)
}
defer resp.Body.Close()
body, err := httputil.DumpResponse(resp, dumpBody)
if err != nil {
log.Fatal("Cannont dump the body of HTTP response", err)
}
return body, resp.StatusCode, err
}
func saveRespToFile(file string, resp []byte) {
err := ioutil.WriteFile(file, resp, 0644)
if err != nil {
panic(err)
}
}
func loadRespFromFile(file string) []byte {
resp, err := ioutil.ReadFile(file)
if err != nil {
panic(err)
}
return resp
}
// ReadResp : reads response from the cached HTTP query.
func ReadResp(fullResp []byte) ([]byte, string, error) {
r := bufio.NewReader(bytes.NewReader(fullResp))
resp, err := http.ReadResponse(r, nil)
if err != nil {
log.Printf("%v\n%s", err, fullResp)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("%v\n%s", err, resp.Body)
}
linkHeader := resp.Header.Get("Link")
return body, linkHeader, err
}
// MakeCachedHTTPRequest makes a cacheable request to the external URL
// If the request was already made once, it will be not done again,
// but read from the file in temporary folder.
// Currently is was tested only for GET queries
func MakeCachedHTTPRequest(url string, tmpFolder string, debug bool) []byte {
var fullResp []byte
filename := GetFilename(url)
filepath := filename
if tmpFolder != "" {
filepath = tmpFolder + "/" + filename
}
if _, err := os.Stat(filepath); os.IsNotExist(err) {
if debug == true {
fmt.Println("HTTP query: " + url)
}
resp, statusCode, err := MakeHTTPRequest(url)
if err != nil {
panic(err)
}
if statusCode == 403 {
log.Fatalf("Looks like the rate limit is exceeded, please try again in 60 minutes. Or make a pull request with authentification feature.")
} else if statusCode == 202 {
log.Printf("Server need some time to prepare request. Trying again.")
time.Sleep(2 * time.Second)
return MakeCachedHTTPRequest(url, tmpFolder, debug)
} else if statusCode != 200 {
log.Fatalf("The status code of URL %s is not OK : %d", url, statusCode)
}
saveRespToFile(filepath, resp)
fullResp = loadRespFromFile(filepath)
} else {
if debug == true {
fmt.Println("Loaded results directly from: " + filepath)
}
fullResp = loadRespFromFile(filepath)
}
return fullResp
}