Use JSONL format for aggregated scans (#233)

stats/stats.go

@@ -1,7 +1,9 @@
 package stats
 
 import (
+	"bufio"
 	"encoding/json"
+	"log"
 	"net/http"
 	"os"
 	"time"
@@ -21,38 +23,42 @@ type Store interface {
 // of the web's top domains
 const topDomainsSource = "TOP_DOMAINS"
 
-// Import imports JSON list of aggregated scans from a remote server to the
-// datastore.
-func Import(store Store) {
+// Import imports aggregated scans from a remote server to the datastore.
+// Expected format is JSONL (newline-separated JSON objects).
+func Import(store Store) error {
 	statsURL := os.Getenv("REMOTE_STATS_URL")
 	resp, err := http.Get(statsURL)
 	if err != nil {
-		raven.CaptureError(err, nil)
-		return
+		return err
 	}
 	defer resp.Body.Close()
 
-	var agScans []checker.AggregatedScan
-	decoder := json.NewDecoder(resp.Body)
-	err = decoder.Decode(&agScans)
-	if err != nil {
-		raven.CaptureError(err, nil)
-		return
-	}
-	for _, a := range agScans {
+	s := bufio.NewScanner(resp.Body)
+	for s.Scan() {
+		var a checker.AggregatedScan
+		err := json.Unmarshal(s.Bytes(), &a)
+		if err != nil {
+			return err
+		}
 		a.Source = topDomainsSource
-		err := store.PutAggregatedScan(a)
+		err = store.PutAggregatedScan(a)
 		if err != nil {
-			raven.CaptureError(err, nil)
+			return err
 		}
 	}
+	if err := s.Err(); err != nil {
+		return err
+	}
+	return nil
 }
 
 // ImportRegularly runs Import to import aggregated stats from a remote server at regular intervals.
 func ImportRegularly(store Store, interval time.Duration) {
 	for {
 		<-time.After(interval)
-		Import(store)
+		err := Import(store)
+		log.Println(err)
+		raven.CaptureError(err, nil)
 	}
 }
 

stats/stats_test.go

@@ -45,12 +45,17 @@ func TestImport(t *testing.T) {
 	}
 	ts := httptest.NewServer(
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			json.NewEncoder(w).Encode(agScans)
+			enc := json.NewEncoder(w)
+			enc.Encode(agScans[0])
+			enc.Encode(agScans[1])
 		}),
 	)
 	os.Setenv("REMOTE_STATS_URL", ts.URL)
 	store := mockAgScanStore{}
-	Import(&store)
+	err := Import(&store)
+	if err != nil {
+		t.Fatal(err)
+	}
 	for i, want := range agScans {
 		got := store[i]
 		// Times must be compared with Time.Equal, so we can't reflect.DeepEqual.