/
raw-downloader.go
77 lines (64 loc) · 1.55 KB
/
raw-downloader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
package main
import (
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
)
func downloadRawData() error {
os.RemoveAll(RAW_DIR)
err := downloadCldrData()
if err != nil {
return err
}
err = downloadW3ContentLanguage()
return err
}
func downloadCldrData() error {
// Fetch data from CLDR repository
repos := []string{
"https://github.com/unicode-cldr/cldr-dates-full.git",
"https://github.com/unicode-cldr/cldr-core.git",
"https://github.com/unicode-cldr/cldr-rbnf.git",
}
for _, repo := range repos {
dirName := strings.TrimSuffix(path.Base(repo), ".git")
dstDir := filepath.Join(RAW_DIR, dirName)
log.Info().Msgf("cloning %s from %s", dirName, repo)
_, err := git.PlainClone(dstDir, false, &git.CloneOptions{
URL: repo,
Depth: 1,
SingleBranch: true,
ReferenceName: plumbing.NewTagReferenceName(CLDR_VERSION)})
if err != nil {
return err
}
}
return nil
}
func downloadW3ContentLanguage() error {
// Prepare dst dir
dstDir := filepath.Join(RAW_DIR, "w3techs")
err := os.MkdirAll(dstDir, os.ModePerm)
if err != nil {
return err
}
// Download page
url := "https://w3techs.com/technologies/overview/content_language"
resp, err := http.DefaultClient.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
// Save to file
bt, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
dstPath := filepath.Join(dstDir, "content_language.html")
return os.WriteFile(dstPath, bt, os.ModePerm)
}