-
Notifications
You must be signed in to change notification settings - Fork 0
/
metadata.go
95 lines (83 loc) · 2.08 KB
/
metadata.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
package main
import (
"encoding/json"
"fmt"
"log"
_ "github.com/mattn/go-sqlite3"
)
type Language struct {
Name string
EscapedName string
CountOfRepos int64
CountOfStars int64
Pages int64
}
type Metadata struct {
CountOfAllRepos int64
AllReposPages int64
CountOfAllStars int64
Languages []Language
}
func numberOfPages(items int64) int64 {
return (items + JSON_PAGINATION_PAGE_SIZE - 1) / JSON_PAGINATION_PAGE_SIZE
}
func saveMetadata() {
// Query for count of all repos
var countOfAllRepos int64
err := db.QueryRow("SELECT COUNT(*) FROM ActiveRepo").Scan(&countOfAllRepos)
if err != nil {
log.Fatal(err)
}
// Query for count of all stars
var countOfAllStars int64
err = db.QueryRow("SELECT SUM(Stargazers) FROM ActiveRepo").Scan(&countOfAllStars)
if err != nil {
log.Fatal(err)
}
// Query for count of repos and stars per language
rows, err := db.Query(`
SELECT Language, SUM(Stargazers), COUNT(*)
FROM ActiveRepo
GROUP BY Language
ORDER BY COUNT(*) DESC, Name
`)
if err != nil {
log.Fatal(err)
}
defer closeOrPanic(rows)
// Store the languages and their counts in a slice
var languages []Language
for rows.Next() {
var languageName string
var countOfStars int64
var countOfRepos int64
err := rows.Scan(&languageName, &countOfStars, &countOfRepos)
if err != nil {
log.Fatal(err)
}
languages = append(languages, Language{
Name: languageName,
EscapedName: escapeLanguageName(languageName),
CountOfRepos: countOfRepos,
CountOfStars: countOfStars,
Pages: numberOfPages(countOfRepos),
})
}
if err = rows.Err(); err != nil {
log.Fatal(err)
}
// Create the Metadata struct and populate it with the extracted data
data := Metadata{
CountOfAllRepos: countOfAllRepos,
CountOfAllStars: countOfAllStars,
AllReposPages: numberOfPages(countOfAllRepos),
Languages: languages,
}
// Marshal the data to JSON format
jsonData, err := json.Marshal(data)
if err != nil {
log.Fatal(err)
}
fileSaveWaitGroup.Add(1)
go saveDataToGzipFile(fmt.Sprintf("%s/metadata", *outputDir), jsonData)
}