/
main.go
118 lines (93 loc) · 2.73 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
package main
import (
"io/ioutil"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/imroc/req/v3"
jsoniter "github.com/json-iterator/go"
"github.com/labstack/gommon/log"
"github.com/mylukin/EchoPilot/helper"
"github.com/mylukin/EchoPilot/service/emoji"
)
var json = jsoniter.ConfigCompatibleWithStandardLibrary
var reqClient *req.Client
type lookup struct {
Name string
URL string
}
func init() {
reqClient = req.C()
reqClient.SetTimeout(5 * time.Second)
}
func main() {
log.Info("Updating Emoji Definition using Emojipedia…")
// Grab the latest Apple Emoji Definitions
res := GetBodyByURL("https://emojipedia.org/apple/")
// Load the Apple Emoji HTML page into goquery so that we
// can query the DOM
doc, err := goquery.NewDocumentFromResponse(res.Response)
if err != nil {
log.Panic(err)
}
// Create a channel for lookups so that we can do this async
lookups := make(chan lookup)
go func() {
// Find all emojis on the page
doc.Find("ul.emoji-grid li").Each(func(i int, s *goquery.Selection) {
// For each item found, get the band and title
emojiPage, _ := s.Find("a").Attr("href")
title, _ := s.Find("img").Attr("title")
log.Infof("Adding Emoji %d to lookups: %s - %s", i, title, emojiPage)
// Add this specific emoji to the lookups to complete
lookups <- lookup{
Name: title,
URL: "https://emojipedia.org" + emojiPage,
}
})
close(lookups)
}()
emojis := map[string]emoji.Emoji{}
// Process a lookup
for lookup := range lookups {
log.Infof("Looking up %s", lookup.Name)
// Grab the emojipedia page for this emoji
res := GetBodyByURL(lookup.URL)
// Create a new goquery reader
doc, err := goquery.NewDocumentFromResponse(res.Response)
if err != nil {
log.Panic(err)
}
// Grab the emoji from the "Copy emoji" input field on the HTML page
emojiString, _ := doc.Find(".copy-paste input[type=text]").Attr("value")
// Convert the raw Emoji value to our hex key
hexString := helper.StringToHexKey(emojiString)
// Add this emoji to our map
emojis[hexString] = emoji.Emoji{
Key: hexString,
Value: emojiString,
Descriptor: lookup.Name,
}
// Print our progress to the console
log.Info(emojis[hexString])
}
// Marshal the emojis map as JSON and write to the data directory
s, _ := json.MarshalIndent(emojis, "", " ")
ioutil.WriteFile("../emoji.json", []byte(s), 0644)
}
// GetBodyByURL is get body by url
func GetBodyByURL(url string) *req.Response {
retryNum := 0
retry:
log.Infof("Get: %s", url)
res, err := reqClient.R().Get(url)
if err != nil {
if retryNum < 5 {
retryNum++
log.Infof(`Retry (%d) after 5 seconds: %s, Error: %s`, retryNum, url, err)
time.Sleep(5 * time.Second)
goto retry
}
log.Error(err)
}
return res
}