Lint, tweaks

checker/cmd/starttls-check/cmd.go

@@ -41,7 +41,7 @@ func main() {
 	c := checker.Checker{
 		Cache: checker.MakeSimpleCache(10 * time.Minute),
 	}
-	w := DomainWriter{}
+	w := domainWriter{}
 
 	if *domain != "" {
 		// Handle single domain and return
@@ -74,9 +74,9 @@ func main() {
 	c.CheckCSV(domainReader, &w, 0)
 }
 
-type DomainWriter struct{}
+type domainWriter struct{}
 
-func (w DomainWriter) HandleDomain(r checker.DomainResult) {
+func (w domainWriter) HandleDomain(r checker.DomainResult) {
 	b, err := json.Marshal(r)
 	if err != nil {
 		fmt.Println(err)

checker/totals.go

@@ -7,9 +7,6 @@ import (
 	"time"
 )
 
-// Sources for aggregated domain statistics
-const MajesticMillion = "majestic-million"
-
 // DomainTotals compiled aggregated stats across domains.
 // Implements ResultHandler.
 type DomainTotals struct {
@@ -21,9 +18,9 @@ type DomainTotals struct {
 	MTASTSEnforce []string
 }
 
-// Add the result of a single domain check to aggregated stats.
+// HandleDomain adds the result of a single domain scan to aggregated stats.
 func (t *DomainTotals) HandleDomain(r DomainResult) {
-	t.Attempted += 1
+	t.Attempted++
 	// Show progress.
 	if t.Attempted%1000 == 0 {
 		log.Printf("%+v\n", t)
@@ -33,7 +30,7 @@ func (t *DomainTotals) HandleDomain(r DomainResult) {
 	if r.Status > 4 {
 		return
 	}
-	t.Connected += 1
+	t.Connected++
 	if r.MTASTSResult != nil {
 		switch r.MTASTSResult.Mode {
 		case "enforce":
@@ -52,7 +49,7 @@ type ResultHandler interface {
 
 const poolSize = 16
 
-// CheckList runs checks on a list of domains, processing the results according
+// CheckCSV runs the checker on a csv of domains, processing the results according
 // to resultHandler.
 func (c *Checker) CheckCSV(domains *csv.Reader, resultHandler ResultHandler, domainColumn int) {
 	work := make(chan string)

db/db.go

@@ -39,6 +39,7 @@ type Database interface {
 	PutHostnameScan(string, checker.HostnameResult) error
 	// Gets counts per day of hosts supporting MTA-STS adoption.
 	GetMTASTSStats() (models.TimeSeries, error)
+	// Store aggregated statistics from multiple scans
 	PutDomainTotals(checker.DomainTotals) error
 	ClearTables() error
 }

db/sqldb.go

@@ -352,6 +352,7 @@ func (db *SQLDatabase) PutHostnameScan(hostname string, result checker.HostnameR
 	return err
 }
 
+// PutDomainTotals writes writes aggregated statistics from multiple scans to the database.
 func (db *SQLDatabase) PutDomainTotals(totals checker.DomainTotals) error {
 	_, err := db.conn.Exec("INSERT INTO domain_totals(time, source, attempted, connected, mta_sts_testing, mta_sts_enforce) VALUES($1, $2, $3, $4, $5, $6)",
 		totals.Time, totals.Source, totals.Attempted, totals.Connected, len(totals.MTASTSTesting), len(totals.MTASTSEnforce))

main.go

@@ -101,7 +101,7 @@ func main() {
 	}
 	if len(os.Args) > 1 {
 		// Use the CLI to run a task other than starting the server.
-		runTask(os.Args[1])
+		runTask(os.Args[1], db)
 		os.Exit(0)
 	}
 	emailConfig, err := makeEmailConfigFromEnv(db)

tasks.go

@@ -8,16 +8,17 @@ import (
 	"time"
 
 	"github.com/EFForg/starttls-backend/checker"
+	"github.com/EFForg/starttls-backend/db"
 )
 
-func runTask(name string) {
+func runTask(name string, db db.Database) {
 	switch name {
 	case "update-stats":
-		updateStats()
+		updateStats(db)
 	}
 }
 
-func updateStats() {
+func updateStats(db db.Database) {
 	resp, err := http.Get("http://downloads.majestic.com/majestic_million.csv")
 	if err != nil {
 		log.Println(err)
@@ -26,11 +27,12 @@ func updateStats() {
 	domains := csv.NewReader(resp.Body)
 	totals := checker.DomainTotals{
 		Time:   time.Now(),
-		Source: checker.MajesticMillion,
+		Source: "majestic-million",
 	}
 	c := checker.Checker{
 		Cache: checker.MakeSimpleCache(10 * time.Minute),
 	}
 	c.CheckCSV(domains, &totals, 2)
+	db.PutDomainTotals(totals)
 	log.Printf("Scans completed, got %+v\n", totals)
 }