This repository has been archived by the owner on Mar 31, 2019. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1
/
users.go
105 lines (98 loc) · 1.91 KB
/
users.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
package main
import (
"database/sql"
"fmt"
"github.com/kjk/stackoverflow"
"github.com/lib/pq"
)
func importUsersIntoDB(r *stackoverflow.Reader, db *sql.DB) (int, error) {
txn, err := db.Begin()
if err != nil {
return 0, err
}
stmt, err := txn.Prepare(pq.CopyIn("users",
"id",
"reputation",
"creation_date",
"display_name",
"last_access_date",
"website_url",
"location",
"about_me",
"views",
"up_votes",
"down_votes",
"age",
"account_id",
"profile_image_url",
))
defer func() {
if txn != nil {
txn.Rollback()
}
}()
if err != nil {
err = fmt.Errorf("txt.Prepare() failed with %s", err)
return 0, err
}
n := 0
for r.Next() {
u := &r.User
_, err = stmt.Exec(
u.ID,
u.Reputation,
u.CreationDate,
toStringPtr(u.DisplayName),
toTimePtr(u.LastAccessDate),
toStringPtr(u.WebsiteURL),
toStringPtr(u.Location),
toStringPtr(u.AboutMe),
u.Views,
u.UpVotes,
u.DownVotes,
u.Age,
u.AccountID,
toStringPtr(u.ProfileImageURL),
)
if err != nil {
LogVerbosef("n: %+v\n", u)
err = fmt.Errorf("stmt.Exec() failed with %s", err)
return 0, err
}
n++
}
if err = r.Err(); err != nil {
return 0, err
}
_, err = stmt.Exec()
if err != nil {
err = fmt.Errorf("stmt.Exec() failed with %s", err)
return 0, err
}
err = stmt.Close()
stmt = nil
if err != nil {
err = fmt.Errorf("stmt.Close() failed with %s", err)
return 0, err
}
err = txn.Commit()
txn = nil
if err != nil {
err = fmt.Errorf("txn.Commit() failed with %s", err)
return 0, err
}
return n, nil
}
func importUsers(siteName string, db *sql.DB) (int, error) {
reader, err := getStackOverflowReader(siteName, "Users")
if err != nil {
return 0, err
}
defer reader.Close()
r, err := stackoverflow.NewUsersReader(reader)
if err != nil {
return 0, fmt.Errorf("stackoverflow.NewUsersReader() failed with %s", err)
}
defer r.Close()
return importUsersIntoDB(r, db)
}