/
main.go
291 lines (236 loc) · 6.98 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
package main
import (
"encoding/csv"
"flag"
"fmt"
"github.com/schollz/progressbar/v3"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"unicode/utf8"
)
var (
dirFlag = flag.String("dir", "", "Directory to search")
dirConfirm = flag.Bool("dirconfirm", false, "Automatically confirm dir is correct")
msgFile = "messages.csv"
baseUrl = "https://cdn.discordapp.com/emojis/"
fileMode = os.FileMode(0644)
)
type Emoji struct {
name string
id int
strId string
filename string
ext string
}
func main() {
flag.Parse()
// Let the user select a directory
dir := selectDir(true)
dir = formatDir(dir)
fmt.Printf("Searching directory \"%s\"\n", dir)
// Find all messages.csv files in said dir
validFiles := getFileList(dir)
validFilesAmt := len(validFiles)
fmt.Printf("Found %v %s files\n", validFilesAmt, msgFile)
if validFilesAmt == 0 {
fmt.Printf("Couldn't find any %s files, maybe try another directory? "+
"Make sure you are selecting the messages directory which contains c<number> folders, "+
"or a c<number> folder itself. Exiting...", msgFile)
return
}
uniqueEmojis := extractUniqueEmojis(validFiles)
// Download all unique emojis found
emojisDir := dir + "emojis/"
err := os.Mkdir(emojisDir, fileMode)
if err != nil && !strings.HasSuffix(err.Error(), "file exists") {
// The file exists error is fine to ignore
fmt.Printf("Error creating emojis directory: %v\n", err)
return
}
downloadAllEmojis(uniqueEmojis, emojisDir)
}
// downloadAllEmojis will download each emoji to messages/emojis/name-id.ext
func downloadAllEmojis(emojis []Emoji, dir string) {
emojisToDownload := make([]Emoji, 0)
current := 0
skipped := 0
// Make a list of non-downloaded emojis (not found on disk)
// These are split into two for loops to avoid messing with the download progress bar
for _, emoji := range emojis {
path := dir + emoji.filename
if !checkFileExists(path) {
emojisToDownload = append(emojisToDownload, emoji)
} else {
skipped += 1
}
}
total := len(emojisToDownload)
// Download all missing emojis
for _, emoji := range emojisToDownload {
current += 1
path := dir + emoji.filename
err := downloadFile(path, baseUrl+emoji.strId+emoji.ext, current, total)
if err != nil {
fmt.Printf("Error downloading emoji: %v\n", err)
}
}
fmt.Printf("A total of %v emojis were downloaded\n", current)
if skipped != 0 {
fmt.Printf("Skipped %v emojis because they were already downloaded\n", skipped)
}
}
// downloadFile will download a given file from url, and display the progress
func downloadFile(filepath string, url string, current int, total int) error {
req, _ := http.NewRequest("GET", url, nil)
res, _ := http.DefaultClient.Do(req)
f, _ := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, fileMode)
defer f.Close()
count := "(" + strconv.Itoa(current) + "/" + strconv.Itoa(total) + ")"
bar := progressbar.NewOptions(-1,
progressbar.OptionFullWidth(),
progressbar.OptionSetDescription("downloading emojis "+count),
progressbar.OptionClearOnFinish(),
)
_, err := io.Copy(io.MultiWriter(f, bar), res.Body)
defer res.Body.Close()
return err
}
// extractUniqueEmojis will search all given files, and extract unique emojis by ID
func extractUniqueEmojis(files []string) []Emoji {
messages := make([]string, 0)
for _, file := range files {
csvF, err := os.Open(file)
if err != nil {
fmt.Printf("Couldn't open file \"%s\": %v\n", file, err)
continue
}
r := csv.NewReader(csvF)
for {
column, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
fmt.Printf("Error reading csv from \"%s\": %v\n", file, err)
break
}
messages = append(messages, column[2]) // Third column is the message content
}
}
fmt.Printf("Found %v messages to parse\n", len(messages))
// Find emojis in all messages now
parsedEmojis := make([]Emoji, 0)
for _, message := range messages {
re := regexp.MustCompile("<(a|):([A-z0-9_]+):([0-9]+)>")
emojis := re.FindAllStringSubmatch(message, -1)
// Parse all emojis in a message. emoji[0] is the full match, 1 is group 1 and so on
for _, emoji := range emojis {
id, err := strconv.Atoi(emoji[3])
if err != nil {
fmt.Printf("Error extracting emoji \"%s\": %v\n", emoji, err)
continue
}
ext := ".png"
if len(emoji[1]) != 0 {
ext = ".gif"
}
filename := emoji[2] + "-" + emoji[3] + ext
parsedEmoji := Emoji{emoji[2], id, emoji[3], filename, ext}
parsedEmojis = append(parsedEmojis, parsedEmoji)
}
}
fmt.Printf("Found %v emojis inside messages\n", len(parsedEmojis))
uniqueEmojis := make([]Emoji, 0)
// Append unique emojis by their ID. This is slow, but I am unsure how to make it faster
for _, pEmoji := range parsedEmojis {
found := false
for _, uEmoji := range uniqueEmojis {
if uEmoji.id == pEmoji.id {
found = true
break
}
}
if !found {
uniqueEmojis = append(uniqueEmojis, pEmoji)
}
}
fmt.Printf("Found %v unique emojis\n", len(uniqueEmojis))
return uniqueEmojis
}
// getFileList will look for all the messages.csv files that exist in dir
func getFileList(dir string) []string {
files := make([]string, 0)
fileInfos, err := ioutil.ReadDir(dir)
if err != nil {
panic(err)
}
for _, f := range fileInfos {
msgFilePath := dir + f.Name() + "/" + msgFile
if !f.IsDir() {
// If f is not a directory, but is a messages.csv
if f.Name() == msgFile {
msgFilePath = dir + "/" + msgFile
if checkFileExists(msgFilePath) {
files = append(files, msgFilePath)
}
}
} else {
// If f is a directory (hopefully the channel directory)
if checkFileExists(msgFilePath) {
files = append(files, msgFilePath)
}
}
}
return files
}
// checkFileExists will check if a certain path exists, and ensures it is not a directory
func checkFileExists(path string) bool {
if f, err := os.Stat(path); err == nil {
return !f.IsDir() // path exists, return true if it is not a directory
} else if os.IsNotExist(err) {
return false // file does not exist
} else { // schrodinger's file
log.Printf("Error: %v", err)
panic(err)
}
return false // go requires we do something here
}
// formatDir will append a / to the end of a dir path if it is missing
func formatDir(dir string) string {
last, _ := utf8.DecodeLastRuneInString(dir)
if last != '/' {
dir += "/"
}
return dir
}
// selectDir will ask the user for a directory and confirm the directory they chose
func selectDir(firstRun bool) string {
// If dirConfirm is selected and there's a dir set, choose it automatically
if *dirConfirm && len(*dirFlag) != 0 {
return *dirFlag
}
var dir string
if firstRun {
dir = *dirFlag
}
if len(*dirFlag) == 0 {
fmt.Printf("Select a directory to scan (use . for current): ")
fmt.Scan(&dir)
}
fmt.Printf("Selected directory: \"%s\"\n", dir)
fmt.Printf("Is this correct? (Y/N): ")
var correct string
fmt.Scan(&correct)
first := strings.ToLower(correct[0:1])
if first != "y" {
fmt.Printf("Selected No, trying again.\n")
return selectDir(false)
}
return dir
}