forked from zmap/zlint
/
main.go
348 lines (311 loc) · 11 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
/*
* ZLint Copyright 2021 Regents of the University of Michigan
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy
* of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"go/format"
"html/template"
"io"
"net"
"net/http"
"os"
"strings"
"time"
log "github.com/sirupsen/logrus"
"github.com/zmap/zlint/v3/util"
)
//nolint:revive
const (
// ICANN_GTLD_JSON is the URL for the ICANN gTLD JSON registry (version 2).
// This registry does not contain ccTLDs but does carry full gTLD information
// needed to determine validity periods.
// See https://www.icann.org/resources/pages/registries/registries-en for more
// information.
ICANN_GTLD_JSON = "https://www.icann.org/resources/registries/gtlds/v2/gtlds.json"
// ICANN_TLDS is the URL for the ICANN list of valid top-level domains
// maintained by the IANA. It contains both ccTLDs and gTLDs but does not
// carry sufficient granularity to determine validity periods.
// See https://www.icann.org/resources/pages/tlds-2012-02-25-en for more
// information.
ICANN_TLDS = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
)
var (
// version is replaced by GoReleaser or `make` using an LDFlags option at
// build time. Here we supply a default value for folks that `go install` or
// `go build` directly from src.
version = "dev-unknown"
// httpClient is a http.Client instance configured with timeouts.
httpClient = &http.Client{
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 15 * time.Second,
KeepAlive: 15 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
}
// gTLDMapTemplate is a template that produces a Golang source code file in
// the "util" package containing a single member variable, a map of strings to
// `util.GTLDPeriod` objects called `tldMap`.
gTLDMapTemplate = template.Must(template.New("gTLDMapTemplate").Parse(
`// Code generated by go generate; DO NOT EDIT.
// This file was generated by zlint-gtld-update.
/*
* ZLint Copyright 2021 Regents of the University of Michigan
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy
* of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package util
var tldMap = map[string]GTLDPeriod{
{{- range .GTLDs }}
"{{ .GTLD }}": {
GTLD: "{{ .GTLD }}",
DelegationDate: "{{ .DelegationDate }}",
RemovalDate: "{{ .RemovalDate }}",
},
{{- end }}
// .onion is a special case and not a general gTLD. However, it is allowed in
// some circumstances in the web PKI so the Zlint gtldMap includes it with
// a delegationDate based on the CABF ballot to allow EV issuance for .onion
// domains: https://cabforum.org/2015/02/18/ballot-144-validation-rules-dot-onion-names/
"onion": {
GTLD: "onion",
DelegationDate: "2015-02-18",
RemovalDate: "",
},
}
`))
printVersion = false
)
// getData fetches the response body bytes from an HTTP get to the provider url,
// or returns an error.
func getData(url string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// Change NewRequest to NewRequestWithContext and pass context it
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("unable to fetch data from %q : %s",
url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code fetching data "+
"from %q : expected status %d got %d",
url, http.StatusOK, resp.StatusCode)
}
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("unexpected error reading response "+
"body from %q : %s",
url, err)
}
return respBody, nil
}
// getTLDData fetches the ICANN_TLDS list and uses the information to build
// and return a list of util.GTLDPeriod objects (or an error if anything fails).
// Since this data source only contains TLD names and not any information
// about delegation/removal all of the returned `util.GTLDPeriod` objects will
// have the DelegationDate "1985-01-01" (matching the `.com` delegation date)
// and no RemovalDate.
func getTLDData() ([]util.GTLDPeriod, error) {
respBody, err := getData(ICANN_TLDS)
if err != nil {
return nil, fmt.Errorf("error getting ICANN TLD list : %s", err)
}
tlds := strings.Split(string(respBody), "\n")
var results []util.GTLDPeriod
for _, tld := range tlds {
// Skip empty lines and the header comment line
if strings.TrimSpace(tld) == "" || strings.HasPrefix(tld, "#") {
continue
}
results = append(results, util.GTLDPeriod{
GTLD: strings.ToLower(tld),
// The TLD list doesn't indicate when any of the TLDs were delegated so
// assume these TLDs were all delegated at the same time as "com".
DelegationDate: "1985-01-01",
})
}
return results, nil
}
// getGTLDData fetches the ICANN_GTLD_JSON and parses it into a list of
// util.GTLDPeriod objects, or returns an error. The gTLDEntries are returned
// as-is and may contain entries that were never delegated from the root DNS.
func getGTLDData() ([]util.GTLDPeriod, error) {
respBody, err := getData(ICANN_GTLD_JSON)
if err != nil {
return nil, fmt.Errorf("error getting ICANN gTLD JSON : %s", err)
}
var results struct {
GTLDs []util.GTLDPeriod
}
if err := json.Unmarshal(respBody, &results); err != nil {
return nil, fmt.Errorf("unexpected error unmarshaling ICANN gTLD JSON response "+
"body from %q : %s",
ICANN_GTLD_JSON, err)
}
return results.GTLDs, nil
}
// delegatedGTLDs filters the provided list of GTLDPeriods removing any entries
// that were never delegated from the root DNS.
func delegatedGTLDs(entries []util.GTLDPeriod) []util.GTLDPeriod {
var results []util.GTLDPeriod
for _, gTLD := range entries {
if gTLD.DelegationDate == "" {
continue
}
results = append(results, gTLD)
}
return results
}
// validateGTLDs checks that all entries have a valid parseable DelegationDate
// string, and if not-empty, a valid parseable RemovalDate string. This function
// assumes an entry with an empty DelegationDate is an error. Use
// `delegatedGTLDs` to filter out entries that were never delegated before
// validating.
func validateGTLDs(entries []util.GTLDPeriod) error {
for _, gTLD := range entries {
// All entries should have a valid delegation date
if _, err := time.Parse(util.GTLDPeriodDateFormat, gTLD.DelegationDate); err != nil {
return err
}
// a gTLD that has not been removed has an empty RemovalDate and that's OK
if _, err := time.Parse(util.GTLDPeriodDateFormat, gTLD.RemovalDate); gTLD.RemovalDate != "" && err != nil {
return err
}
}
return nil
}
// renderGTLDMap fetches the ICANN gTLD data, filters out undelegated entries,
// validates the remaining entries have parseable dates, and renders the
// gTLDMapTemplate to the provided writer using the validated entries (or
// returns an error if any of the aforementioned steps fail). It then fetches
// the ICANN TLD data, and uses it to populate any missing entries for ccTLDs.
// These entries will have a default delegationDate because the data source is
// not specific enough to provide one. The produced output text is a Golang
// source code file in the `util` package that contains a single map variable
// containing GTLDPeriod objects created with the ICANN data.
func renderGTLDMap(writer io.Writer) error {
// Get all of ICANN's gTLDs including ones that haven't been delegated.
allGTLDs, err := getGTLDData()
if err != nil {
return err
}
// Filter out the non-delegated gTLD entries
delegatedGTLDs := delegatedGTLDs(allGTLDs)
// Validate that all of the delegated gTLDs have correct dates
if err := validateGTLDs(delegatedGTLDs); err != nil {
return err
}
// Get all of the TLDs. This data source doesn't provide delegationDates and
// so we only want to use it to populate missing entries in `delegatedGTLDs`,
// not to replace any existing entries that have more specific information
// about the validity period for the TLD.
allTLDs, err := getTLDData()
if err != nil {
return err
}
tldMap := make(map[string]util.GTLDPeriod)
// Deduplicate delegatedGTLDs into the tldMap first
for _, tld := range delegatedGTLDs {
tldMap[tld.GTLD] = tld
}
// Then populate any missing entries from the allTLDs list
for _, tld := range allTLDs {
if _, found := tldMap[tld.GTLD]; !found {
tldMap[tld.GTLD] = tld
}
}
templateData := struct {
GTLDs map[string]util.GTLDPeriod
}{
GTLDs: tldMap,
}
// Render the gTLD map to a buffer with the delegated gTLD data
var buf bytes.Buffer
if err := gTLDMapTemplate.Execute(&buf, templateData); err != nil {
return err
}
// format the buffer so it won't trip up the `gofmt_test.go` checks
formatted, err := format.Source(buf.Bytes())
if err != nil {
return err
}
// Write the formatted buffer to the writer
_, err = writer.Write(formatted)
if err != nil {
return err
}
return nil
}
// init sets up command line flags
func init() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "ZLint version %s\n\n", version)
fmt.Fprintf(os.Stderr, "Usage: %s [flags]\n", os.Args[0])
flag.PrintDefaults()
}
flag.BoolVar(&printVersion, "version", false, "Print ZLint version and exit")
flag.Parse()
log.SetLevel(log.InfoLevel)
}
// main handles rendering a gTLD map to either standard out (when no argument is
// provided) or to the provided filename. If an error occurs it is printed to
// standard err and the program terminates with a non-zero exit status.
func main() {
errQuit := func(err error) {
fmt.Fprintf(os.Stderr, "error updating gTLD map: %s\n", err)
os.Exit(1)
}
if printVersion {
fmt.Printf("ZLint version %s\n", version)
return
}
// Default to writing to standard out
writer := os.Stdout
if flag.NArg() > 0 {
// If a filename is specified as a command line flag then open it (creating
// if needed), truncate the existing contents, and use the file as the
// writer instead of standard out
filename := flag.Args()[0]
f, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0664)
if err != nil {
errQuit(err)
}
defer f.Close()
writer = f
}
if err := renderGTLDMap(writer); err != nil {
errQuit(err)
}
}