-
Notifications
You must be signed in to change notification settings - Fork 3
/
main.go
263 lines (242 loc) · 9.4 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
package main
import (
"database/sql"
"flag"
"fmt"
"log"
"os"
_ "github.com/mattn/go-sqlite3"
"github.com/respawner/peeringdb"
)
const dbSchema = `
CREATE TABLE peeringdb_facility (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
address1 varchar(255) NOT NULL,
address2 varchar(255) NOT NULL,
city varchar(255) NOT NULL,
state varchar(255) NOT NULL,
zipcode varchar(48) NOT NULL,
country varchar(2) NOT NULL,
name varchar(255) NOT NULL UNIQUE,
website varchar(255) NOT NULL,
clli varchar(18) NOT NULL,
rencode varchar(18) NOT NULL,
npanxx varchar(21) NOT NULL,
notes text NOT NULL,
org_id integer NOT NULL REFERENCES peeringdb_organization (id)
);
CREATE TABLE peeringdb_ix (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
name varchar(64) NOT NULL UNIQUE,
name_long varchar(254) NOT NULL,
city varchar(192) NOT NULL,
country varchar(2) NOT NULL,
notes text NOT NULL,
region_continent varchar(255) NOT NULL,
media varchar(128) NOT NULL,
proto_unicast bool NOT NULL,
proto_multicast bool NOT NULL,
proto_ipv6 bool NOT NULL,
website varchar(255) NOT NULL,
url_stats varchar(255) NOT NULL,
tech_email varchar(254) NOT NULL,
tech_phone varchar(192) NOT NULL,
policy_email varchar(254) NOT NULL,
policy_phone varchar(192) NOT NULL,
org_id integer NOT NULL REFERENCES peeringdb_organization (id)
);
CREATE TABLE peeringdb_ix_facility (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
ix_id integer NOT NULL REFERENCES peeringdb_ix (id),
fac_id integer NOT NULL REFERENCES peeringdb_facility (id),
UNIQUE (ix_id, fac_id)
);
CREATE TABLE peeringdb_ixlan (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
name varchar(255) NOT NULL,
descr text NOT NULL,
mtu integer unsigned NULL,
vlan integer unsigned NULL,
dot1q_support bool NOT NULL,
rs_asn integer unsigned NULL,
arp_sponge varchar(17) NULL,
ix_id integer NOT NULL REFERENCES peeringdb_ix (id)
);
CREATE TABLE peeringdb_ixlan_prefix (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
notes varchar(255) NOT NULL,
protocol varchar(64) NOT NULL,
prefix varchar(43) NOT NULL UNIQUE,
ixlan_id integer NOT NULL REFERENCES peeringdb_ixlan (id)
);
CREATE TABLE peeringdb_network (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
asn integer unsigned NOT NULL UNIQUE,
name varchar(255) NOT NULL UNIQUE,
aka varchar(255) NOT NULL,
irr_as_set varchar(255) NOT NULL,
website varchar(255) NOT NULL,
looking_glass varchar(255) NOT NULL,
route_server varchar(255) NOT NULL,
notes text NOT NULL,
notes_private text NOT NULL,
info_traffic varchar(39) NOT NULL,
info_ratio varchar(45) NOT NULL,
info_scope varchar(39) NOT NULL,
info_type varchar(60) NOT NULL,
info_prefixes4 integer unsigned NULL,
info_prefixes6 integer unsigned NULL,
info_unicast bool NOT NULL,
info_multicast bool NOT NULL,
info_ipv6 bool NOT NULL,
policy_url varchar(255) NOT NULL,
policy_general varchar(72) NOT NULL,
policy_locations varchar(72) NOT NULL,
policy_ratio bool NOT NULL,
policy_contracts varchar(36) NOT NULL,
org_id integer NOT NULL REFERENCES peeringdb_organization (id)
);
CREATE TABLE peeringdb_network_contact (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
role varchar(27) NOT NULL,
visible varchar(64) NOT NULL,
name varchar(254) NOT NULL,
phone varchar(100) NOT NULL,
email varchar(254) NOT NULL,
url varchar(255) NOT NULL,
net_id integer NOT NULL REFERENCES peeringdb_network (id)
);
CREATE TABLE peeringdb_network_facility (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
local_asn integer unsigned NULL,
avail_sonet bool NOT NULL,
avail_ethernet bool NOT NULL,
avail_atm bool NOT NULL,
net_id integer NOT NULL REFERENCES peeringdb_network (id),
fac_id integer NOT NULL REFERENCES peeringdb_facility (id),
UNIQUE (net_id, fac_id, local_asn)
);
CREATE TABLE peeringdb_network_ixlan (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
asn integer unsigned NOT NULL,
ipaddr4 varchar(39) NULL,
ipaddr6 varchar(39) NULL,
is_rs_peer bool NOT NULL,
notes varchar(255) NOT NULL,
speed integer unsigned NOT NULL,
net_id integer NOT NULL REFERENCES peeringdb_network (id),
ixlan_id integer NOT NULL REFERENCES peeringdb_ixlan (id)
);
CREATE TABLE peeringdb_organization (
id integer NOT NULL PRIMARY KEY AUTOINCREMENT,
status varchar(255) NOT NULL,
created datetime NOT NULL,
updated datetime NOT NULL,
version integer NOT NULL,
address1 varchar(255) NOT NULL,
address2 varchar(255) NOT NULL,
city varchar(255) NOT NULL,
state varchar(255) NOT NULL,
zipcode varchar(48) NOT NULL,
country varchar(2) NOT NULL,
name varchar(255) NOT NULL UNIQUE,
website varchar(255) NOT NULL,
notes text NOT NULL
);
CREATE INDEX peeringdb_facility_org_id ON peeringdb_facility (org_id);
CREATE INDEX peeringdb_ix_org_id ON peeringdb_ix (org_id);
CREATE INDEX peeringdb_ix_facility_ix_id ON peeringdb_ix_facility (ix_id);
CREATE INDEX peeringdb_ix_facility_fac_id ON peeringdb_ix_facility (fac_id);
CREATE INDEX peeringdb_ixlan_ix_id ON peeringdb_ixlan (ix_id);
CREATE INDEX peeringdb_ixlan_prefix_ixlan_id ON peeringdb_ixlan_prefix (ixlan_id);
CREATE INDEX peeringdb_network_org_id ON peeringdb_network (org_id);
CREATE INDEX peeringdb_network_contact_net_id ON peeringdb_network_contact (net_id);
CREATE INDEX peeringdb_network_facility_net_id ON peeringdb_network_facility (net_id);
CREATE INDEX peeringdb_network_facility_fac_id ON peeringdb_network_facility (fac_id);
CREATE INDEX peeringdb_network_ixlan_ixlan_id ON peeringdb_network_ixlan (ixlan_id);
CREATE INDEX peeringdb_network_ixlan_net_id ON peeringdb_network_ixlan (net_id);`
func main() {
databaseFile := "./peeringdb.db"
initialSynchronization := true
fullRequested := flag.Bool("full", false,
"Request a full synchronization (needed to remove old entries)")
flag.Parse()
// The database already exists, assume it is not the first synchronization
if _, err := os.Stat(databaseFile); err == nil {
if *fullRequested {
err = os.Remove(databaseFile)
if err != nil {
log.Fatal(err)
} else {
fmt.Println("Full synchronization requested, the local database has been removed.")
}
} else {
initialSynchronization = false
}
}
// Open the SQLite database, will create it if needed
db, err := sql.Open("sqlite3", databaseFile)
if err != nil {
log.Fatal(err)
}
defer db.Close()
// Prepare to query the API and the synchronization
sync := synchronization{peeringdb.NewAPI(), db}
if initialSynchronization {
// First time, create the database schema
_, err = db.Exec(dbSchema)
if err != nil {
log.Printf("%q: %s\n", err, dbSchema)
return
}
}
fmt.Println("Starting PeeringDB synchronization...")
// Synchronize all objects
sync.synchronizeOrganizations()
sync.synchronizeFacilities()
sync.synchronizeNetworks()
sync.synchronizeInternetExchanges()
sync.synchronizeInternetExchangeFacilities()
sync.synchronizeInternetLANs()
sync.synchronizeInternetPrefixes()
sync.synchronizeNetworkContacts()
sync.synchronizeNetworkFacilities()
sync.synchronizeNetworkInternetExchangeLANs()
fmt.Println("PeeringDB fully synchronized.")
}