Merge be8794a into 36b4432

checker/totals.go

@@ -22,6 +22,12 @@ type AggregatedScan struct {
 	MTASTSEnforceList []string
 }
 
+// PercentMTASTS returns the fraction of domains with MXs that support
+// MTA-STS, represented as a float between 0 and 1.
+func (a AggregatedScan) PercentMTASTS() float64 {
+	return (float64(a.MTASTSTesting) + float64(a.MTASTSEnforce)) / float64(a.WithMXs)
+}
+
 // HandleDomain adds the result of a single domain scan to aggregated stats.
 func (a *AggregatedScan) HandleDomain(r DomainResult) {
 	a.Attempted++

db/db.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/EFForg/starttls-backend/checker"
 	"github.com/EFForg/starttls-backend/models"
+	"github.com/EFForg/starttls-backend/stats"
 )
 
 // Database interface: These are the things that the Database should be able to do.
@@ -33,8 +34,10 @@ type Database interface {
 	PutHostnameScan(string, checker.HostnameResult) error
 	// Writes an aggregated scan to the database
 	PutAggregatedScan(checker.AggregatedScan) error
-	// Gets counts per day of hosts supporting MTA-STS adoption.
-	GetMTASTSStats() (models.TimeSeries, error)
+	// Gets counts per day of hosts supporting MTA-STS for a given source.
+	GetMTASTSStats(string) (stats.Series, error)
+	// Gets counts per day of hosts scanned by this app supporting MTA-STS adoption.
+	GetMTASTSLocalStats() (stats.Series, error)
 	// Upserts domain state.
 	PutDomain(models.Domain) error
 	// Retrieves state of a domain

db/scripts/init_tables.sql

@@ -85,7 +85,7 @@ CREATE TABLE IF NOT EXISTS aggregated_scans
     time            TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     source          TEXT NOT NULL,
     attempted       INTEGER DEFAULT 0,
-    with_mxs         INTEGER DEFAULT 0,
+    with_mxs        INTEGER DEFAULT 0,
     mta_sts_testing INTEGER DEFAULT 0,
     mta_sts_enforce INTEGER DEFAULT 0
 );
@@ -103,3 +103,4 @@ ALTER TABLE IF EXISTS aggregated_scans ADD COLUMN IF NOT EXISTS with_mxs INTEGER
 
 ALTER TABLE domains ADD COLUMN IF NOT EXISTS mta_sts BOOLEAN DEFAULT FALSE;
 
+ALTER TABLE aggregated_scans ADD UNIQUE (time, source);

db/sqldb.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/EFForg/starttls-backend/checker"
 	"github.com/EFForg/starttls-backend/models"
+	"github.com/EFForg/starttls-backend/stats"
 
 	// Imports postgresql driver for database/sql
 	_ "github.com/lib/pq"
@@ -116,12 +117,33 @@ func (db *SQLDatabase) PutScan(scan models.Scan) error {
 	return err
 }
 
-// GetMTASTSStats returns statistics about MTA-STS adoption over a rolling
-// 14-day window.
-// Returns a map with
+// GetMTASTSStats returns statistics about a MTA-STS adoption from a single
+// source domains to check.
+func (db *SQLDatabase) GetMTASTSStats(source string) (stats.Series, error) {
+	rows, err := db.conn.Query(
+		"SELECT time, with_mxs, mta_sts_testing, mta_sts_enforce FROM aggregated_scans WHERE source=$1", source)
+	if err != nil {
+		return stats.Series{}, err
+	}
+	defer rows.Close()
+	series := stats.Series{}
+	for rows.Next() {
+		var a checker.AggregatedScan
+		if err := rows.Scan(&a.Time, &a.WithMXs, &a.MTASTSTesting, &a.MTASTSEnforce); err != nil {
+			return stats.Series{}, err
+		}
+		series[a.Time.UTC()] = a.PercentMTASTS()
+	}
+	return series, nil
+}
+
+// GetMTASTSLocalStats returns statistics about MTA-STS adoption in
+// user-initiated scans over a rolling 14-day window.  Returns a map with:
 //  key: the final day of a two-week window. Windows last until EOD.
 //  value: the percent of scans supporting MTA-STS in that window
-func (db *SQLDatabase) GetMTASTSStats() (models.TimeSeries, error) {
+// @TODO write a simpler query that gets caches totals in the the
+// `aggregated_scans` table at the end of each 14-day period
+func (db *SQLDatabase) GetMTASTSLocalStats() (stats.Series, error) {
 	// "day" represents truncated date (ie beginning of day), but windows should
 	// include the full day, so we add a day when querying timestamps.
 	// Getting the most recent 31 days for now, we can set the start date to the
@@ -149,10 +171,10 @@ func (db *SQLDatabase) GetMTASTSStats() (models.TimeSeries, error) {
 	}
 	defer rows.Close()
 
-	ts := make(map[time.Time]float32)
+	ts := make(map[time.Time]float64)
 	for rows.Next() {
 		var t time.Time
-		var count float32
+		var count float64
 		if err := rows.Scan(&t, &count); err != nil {
 			return nil, err
 		}
@@ -287,6 +309,7 @@ func (db SQLDatabase) ClearTables() error {
 		fmt.Sprintf("DELETE FROM %s", db.cfg.DbTokenTable),
 		fmt.Sprintf("DELETE FROM %s", "hostname_scans"),
 		fmt.Sprintf("DELETE FROM %s", "blacklisted_emails"),
+		fmt.Sprintf("DELETE FROM %s", "aggregated_scans"),
 		fmt.Sprintf("ALTER SEQUENCE %s_id_seq RESTART WITH 1", db.cfg.DbScanTable),
 	})
 }
@@ -384,7 +407,8 @@ func (db *SQLDatabase) PutHostnameScan(hostname string, result checker.HostnameR
 func (db *SQLDatabase) PutAggregatedScan(a checker.AggregatedScan) error {
 	_, err := db.conn.Exec(`INSERT INTO
 		aggregated_scans(time, source, attempted, with_mxs, mta_sts_testing, mta_sts_enforce)
-		VALUES ($1, $2, $3, $4, $5, $6)`,
+		VALUES ($1, $2, $3, $4, $5, $6)
+		ON CONFLICT (time,source) DO NOTHING`,
 		a.Time, a.Source, a.Attempted, a.WithMXs, a.MTASTSTesting, a.MTASTSEnforce)
 	return err
 }

db/sqldb_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/EFForg/starttls-backend/checker"
 	"github.com/EFForg/starttls-backend/db"
 	"github.com/EFForg/starttls-backend/models"
+	"github.com/EFForg/starttls-backend/stats"
 	"github.com/joho/godotenv"
 )
 
@@ -344,7 +345,53 @@ func TestGetHostnameScan(t *testing.T) {
 	}
 }
 
+func dateMustParse(date string, t *testing.T) time.Time {
+	const shortForm = "2006-Jan-02"
+	parsed, err := time.Parse(shortForm, date)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return parsed
+}
+
 func TestGetMTASTSStats(t *testing.T) {
+	database.ClearTables()
+	may1 := dateMustParse("2019-May-01", t)
+	may2 := dateMustParse("2019-May-02", t)
+	data := []checker.AggregatedScan{
+		checker.AggregatedScan{
+			Time:          may1,
+			Source:        "domains-depot",
+			Attempted:     5,
+			WithMXs:       4,
+			MTASTSTesting: 2,
+			MTASTSEnforce: 1,
+		},
+		checker.AggregatedScan{
+			Time:          may2,
+			Source:        "domains-depot",
+			Attempted:     10,
+			WithMXs:       8,
+			MTASTSTesting: 1,
+			MTASTSEnforce: 3,
+		},
+	}
+	for _, a := range data {
+		err := database.PutAggregatedScan(a)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	result, err := database.GetMTASTSStats("domains-depot")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result[may1] != float64(0.75) || result[may2] != float64(0.5) {
+		t.Errorf("Incorrect MTA-STS stats, got %v", result)
+	}
+}
+
+func TestGetMTASTSLocalStats(t *testing.T) {
 	database.ClearTables()
 	day := time.Hour * 24
 	today := time.Now()
@@ -363,7 +410,7 @@ func TestGetMTASTSStats(t *testing.T) {
 	database.PutScan(s)
 	// Support is shown in the rolling average until the no-support scan is
 	// included.
-	expectStats(models.TimeSeries{
+	expectStats(stats.Series{
 		lastWeek:              100,
 		lastWeek.Add(day):     100,
 		lastWeek.Add(2 * day): 100,
@@ -380,7 +427,7 @@ func TestGetMTASTSStats(t *testing.T) {
 		Timestamp: lastWeek.Add(1 * day),
 	}
 	database.PutScan(s)
-	expectStats(models.TimeSeries{
+	expectStats(stats.Series{
 		lastWeek:              100,
 		lastWeek.Add(day):     100,
 		lastWeek.Add(2 * day): 100,
@@ -397,27 +444,27 @@ func TestGetMTASTSStats(t *testing.T) {
 		Timestamp: lastWeek.Add(6 * day),
 	}
 	database.PutScan(s)
-	expectStats(models.TimeSeries{
+	expectStats(stats.Series{
 		lastWeek:              100,
 		lastWeek.Add(day):     100,
 		lastWeek.Add(2 * day): 100,
 		lastWeek.Add(3 * day): 50,
 		lastWeek.Add(4 * day): 50,
 		lastWeek.Add(5 * day): 50,
-		lastWeek.Add(6 * day): 66.666664,
+		lastWeek.Add(6 * day): 66.66666666666667,
 	}, t)
 }
 
-func expectStats(ts models.TimeSeries, t *testing.T) {
+func expectStats(ts stats.Series, t *testing.T) {
 	// GetMTASTSStats returns dates only (no hours, minutes, seconds). We need
 	// to truncate the expected times for comparison to dates and convert to UTC
 	// to match the database's timezone.
-	expected := make(map[time.Time]float32)
+	expected := make(map[time.Time]float64)
 	for kOld, v := range ts {
 		k := kOld.UTC().Truncate(24 * time.Hour)
 		expected[k] = v
 	}
-	got, err := database.GetMTASTSStats()
+	got, err := database.GetMTASTSLocalStats()
 	if err != nil {
 		t.Fatal(err)
 	}

stats.go

@@ -2,14 +2,16 @@ package main
 
 import (
 	"net/http"
+
+	"github.com/EFForg/starttls-backend/stats"
 )
 
 // Stats returns statistics about MTA-STS adoption over a 14-day rolling window.
 func (api API) Stats(r *http.Request) APIResponse {
 	if r.Method != http.MethodGet {
 		return APIResponse{StatusCode: http.StatusMethodNotAllowed}
 	}
-	stats, err := api.Database.GetMTASTSStats()
+	stats, err := stats.Get(api.Database)
 	if err != nil {
 		return serverError(err.Error())
 	}

stats/stats.go

@@ -0,0 +1,85 @@
+package stats
+
+import (
+	"bufio"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/EFForg/starttls-backend/checker"
+	raven "github.com/getsentry/raven-go"
+)
+
+// Store wraps storage for MTA-STS adoption statistics.
+type Store interface {
+	PutAggregatedScan(checker.AggregatedScan) error
+	GetMTASTSStats(string) (Series, error)
+	GetMTASTSLocalStats() (Series, error)
+}
+
+// Identifier in the DB for aggregated scans we imported from our regular scans
+// of the web's top domains
+const topDomainsSource = "TOP_DOMAINS"
+
+// Import imports aggregated scans from a remote server to the datastore.
+// Expected format is JSONL (newline-separated JSON objects).
+func Import(store Store) error {
+	statsURL := os.Getenv("REMOTE_STATS_URL")
+	resp, err := http.Get(statsURL)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	s := bufio.NewScanner(resp.Body)
+	for s.Scan() {
+		var a checker.AggregatedScan
+		err := json.Unmarshal(s.Bytes(), &a)
+		if err != nil {
+			return err
+		}
+		a.Source = topDomainsSource
+		err = store.PutAggregatedScan(a)
+		if err != nil {
+			return err
+		}
+	}
+	if err := s.Err(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ImportRegularly runs Import to import aggregated stats from a remote server at regular intervals.
+func ImportRegularly(store Store, interval time.Duration) {
+	for {
+		<-time.After(interval)
+		err := Import(store)
+		log.Println(err)
+		raven.CaptureError(err, nil)
+	}
+}
+
+// Series represents some statistic as it changes over time.
+// This will likely be updated when we know what format our frontend charting
+// library prefers.
+type Series map[time.Time]float64
+
+// Get retrieves MTA-STS adoption statistics for user-initiated scans and scans
+// of the top million domains over time.
+func Get(store Store) (map[string]Series, error) {
+	result := make(map[string]Series)
+	series, err := store.GetMTASTSStats(topDomainsSource)
+	if err != nil {
+		return result, err
+	}
+	result["top-million"] = series
+	series, err = store.GetMTASTSLocalStats()
+	if err != nil {
+		return result, err
+	}
+	result["local"] = series
+	return result, err
+}