Get and serve mta-sts adoption timeseries from aggregated scans

checker/totals.go

@@ -22,6 +22,10 @@ type AggregatedScan struct {
 	MTASTSEnforceList []string
 }
 
+func (a AggregatedScan) PercentMTASTS() float64 {
+	return (float64(a.MTASTSTesting) + float64(a.MTASTSEnforce)) / float64(a.WithMXs)
+}
+
 // HandleDomain adds the result of a single domain scan to aggregated stats.
 func (a *AggregatedScan) HandleDomain(r DomainResult) {
 	a.Attempted++

db/db.go

@@ -30,6 +30,8 @@ type Database interface {
 	IsBlacklistedEmail(string) (bool, error)
 	// Retrieves a hostname scan for a particular hostname
 	GetHostnameScan(string) (checker.HostnameResult, error)
+	// Writes an aggregated scan to the database
+	PutAggregatedScan(checker.AggregatedScan) error
 	// Enters a hostname scan.
 	PutHostnameScan(string, checker.HostnameResult) error
 	// Gets counts per day of hosts supporting MTA-STS for a given source.

db/sqldb.go

@@ -118,14 +118,29 @@ func (db *SQLDatabase) PutScan(scan models.Scan) error {
 }
 
 func (db *SQLDatabase) GetMTASTSStats(source string) (stats.Series, error) {
-	return stats.Series{}, nil
+	rows, err := db.conn.Query(
+		"SELECT time, with_mxs, mta_sts_testing, mta_sts_enforce FROM aggregated_scans WHERE source=$1", source)
+	if err != nil {
+		return stats.Series{}, err
+	}
+	defer rows.Close()
+	series := stats.Series{}
+	for rows.Next() {
+		var a checker.AggregatedScan
+		if err := rows.Scan(&a.Time, &a.WithMXs, &a.MTASTSTesting, &a.MTASTSEnforce); err != nil {
+			return stats.Series{}, err
+		}
+		series[a.Time.UTC()] = a.PercentMTASTS()
+	}
+	return series, nil
 }
 
-// GetMTASTSStats returns statistics about MTA-STS adoption over a rolling
-// 14-day window.
-// Returns a map with
+// GetMTASTSLocalStats returns statistics about MTA-STS adoption over a rolling
+// 14-day window.  Returns a map with
 //  key: the final day of a two-week window. Windows last until EOD.
 //  value: the percent of scans supporting MTA-STS in that window
+// @TODO write a simpler query that gets caches totals in the the
+// `aggregated_scans` table at the end of each 14-day period
 func (db *SQLDatabase) GetMTASTSLocalStats() (stats.Series, error) {
 	// "day" represents truncated date (ie beginning of day), but windows should
 	// include the full day, so we add a day when querying timestamps.
@@ -154,10 +169,10 @@ func (db *SQLDatabase) GetMTASTSLocalStats() (stats.Series, error) {
 	}
 	defer rows.Close()
 
-	ts := make(map[time.Time]float32)
+	ts := make(map[time.Time]float64)
 	for rows.Next() {
 		var t time.Time
-		var count float32
+		var count float64
 		if err := rows.Scan(&t, &count); err != nil {
 			return nil, err
 		}
@@ -292,6 +307,7 @@ func (db SQLDatabase) ClearTables() error {
 		fmt.Sprintf("DELETE FROM %s", db.cfg.DbTokenTable),
 		fmt.Sprintf("DELETE FROM %s", "hostname_scans"),
 		fmt.Sprintf("DELETE FROM %s", "blacklisted_emails"),
+		fmt.Sprintf("DELETE FROM %s", "aggregated_scans"),
 		fmt.Sprintf("ALTER SEQUENCE %s_id_seq RESTART WITH 1", db.cfg.DbScanTable),
 	})
 }

db/sqldb_test.go

@@ -345,7 +345,50 @@ func TestGetHostnameScan(t *testing.T) {
 	}
 }
 
+func dateMustParse(date string, t *testing.T) time.Time {
+	const shortForm = "2006-Jan-02"
+	parsed, err := time.Parse(shortForm, date)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return parsed
+}
+
 func TestGetMTASTSStats(t *testing.T) {
+	database.ClearTables()
+	may1 := dateMustParse("2019-May-01", t)
+	may2 := dateMustParse("2019-May-02", t)
+	data := []checker.AggregatedScan{
+		checker.AggregatedScan{
+			Time:          may1,
+			Source:        "domains-depot",
+			Attempted:     5,
+			WithMXs:       4,
+			MTASTSTesting: 2,
+			MTASTSEnforce: 1,
+		},
+		checker.AggregatedScan{
+			Time:          may2,
+			Source:        "domains-depot",
+			Attempted:     10,
+			WithMXs:       8,
+			MTASTSTesting: 1,
+			MTASTSEnforce: 3,
+		},
+	}
+	for _, a := range data {
+		database.PutAggregatedScan(a)
+	}
+	result, err := database.GetMTASTSStats("domains-depot")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result[may1] != float64(0.75) || result[may2] != float64(0.5) {
+		t.Errorf("Incorrect MTA-STS stats, got %v", result)
+	}
+}
+
+func TestGetMTASTSLocalStats(t *testing.T) {
 	database.ClearTables()
 	day := time.Hour * 24
 	today := time.Now()
@@ -413,7 +456,7 @@ func expectStats(ts stats.Series, t *testing.T) {
 	// GetMTASTSStats returns dates only (no hours, minutes, seconds). We need
 	// to truncate the expected times for comparison to dates and convert to UTC
 	// to match the database's timezone.
-	expected := make(map[time.Time]float32)
+	expected := make(map[time.Time]float64)
 	for kOld, v := range ts {
 		k := kOld.UTC().Truncate(24 * time.Hour)
 		expected[k] = v

stats.go

@@ -2,14 +2,16 @@ package main
 
 import (
 	"net/http"
+
+	"github.com/EFForg/starttls-backend/stats"
 )
 
 // Stats returns statistics about MTA-STS adoption over a 14-day rolling window.
 func (api API) Stats(r *http.Request) APIResponse {
 	if r.Method != http.MethodGet {
 		return APIResponse{StatusCode: http.StatusMethodNotAllowed}
 	}
-	stats, err := api.Database.GetMTASTSLocalStats()
+	stats, err := stats.Get(api.Database)
 	if err != nil {
 		return serverError(err.Error())
 	}

stats/import.go

@@ -47,20 +47,21 @@ func ImportRegularly(store Store, interval time.Duration) {
 	}
 }
 
-type Series map[time.Time]float32
+type Series map[time.Time]float64
 
 const topMillionSource = "majestic-million"
 
-func Get(store Store) (r map[string]Series, err error) {
+func Get(store Store) (map[string]Series, error) {
+	result := make(map[string]Series)
 	series, err := store.GetMTASTSStats(topMillionSource)
 	if err != nil {
-		return
+		return result, err
 	}
-	r["top-million"] = series
+	result["top-million"] = series
 	series, err = store.GetMTASTSLocalStats()
 	if err != nil {
-		return
+		return result, err
 	}
-	r["local"] = series
-	return
+	result["local"] = series
+	return result, err
 }

stats_test.go

@@ -37,7 +37,7 @@ func TestGetStats(t *testing.T) {
 		t.Fatal(err)
 	}
 	date := now.UTC().Truncate(24 * time.Hour).Format(time.RFC3339)
-	expected := fmt.Sprintf("\"response\": {\n    \"%v\": 100\n  }", date)
+	expected := fmt.Sprintf("\"%v\": 100", date)
 	if !strings.Contains(string(body), expected) {
 		t.Errorf("Expected %s to contain %s", string(body), expected)
 	}