Roll up flights by day

README.md

@@ -34,7 +34,11 @@ following months:
 * [February 2019](https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2019_2.zip)
 * [March 2019](https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2019_3.zip)
 
-After the data has been loaded, run `sql/updates/mark_active_airports.sql`.
+After the data has been loaded, run the SQL files in `sql/updates`:
+
+```sh
+cat sql/updates/*.sql | mysql -uflightdb -pflightdb -h 127.0.0.1 flightdb
+```
 
 ## Configuration
 

backendA/stats.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"database/sql"
 	"sort"
 	"strings"
@@ -42,110 +41,51 @@ func resolveFlightStatsByAirline(db *sql.DB) graphql.FieldResolveFn {
 				return nil, nil
 			}
 
-			stats, err := airlineFlightInfo(p.Context, db, origin, dest)
+			rows, err := db.QueryContext(p.Context,
+				`SELECT
+					carriers.name AS carrier_name, total_flights, delays_flights, last_flight
+				FROM
+					(
+						SELECT
+							carrier AS carrier_code,
+							SUM(total_flights) AS total_flights,
+							SUM(delayed_flights) AS delays_flights,
+							MAX(date) AS last_flight
+						FROM
+							flights_day
+						WHERE origin=? AND destination=?
+						GROUP BY carrier_code
+					) AS stats
+				INNER JOIN carriers ON carrier_code=carriers.code
+				`,
+				origin, dest)
 			if err != nil {
 				return nil, err
 			}
+			defer rows.Close()
 
-			delays, err := delaysByAirline(p.Context, db, origin, dest)
-			if err != nil {
-				return nil, err
-			}
+			stats := []*airlineStats{}
 
-			for code := range stats {
-				stats[code].OnTimePercentage = (1.0 - float64(delays[code])/float64(stats[code].TotalFlights)) * 100
-			}
+			for rows.Next() {
+				var (
+					row            airlineStats
+					delayedFlights int
+				)
+				err := rows.Scan(&row.Airline, &row.TotalFlights, &delayedFlights, &row.LastFlight)
+				if err != nil {
+					return nil, err
+				}
+
+				row.OnTimePercentage = (1.0 - float64(delayedFlights)/float64(row.TotalFlights)) * 100
 
-			statsRows := make([]*airlineStats, len(stats))
-			i := 0
-			for code := range stats {
-				statsRows[i] = stats[code]
-				i++
+				stats = append(stats, &row)
 			}
 
-			sort.Slice(statsRows, func(i, j int) bool {
-				return statsRows[j].OnTimePercentage < statsRows[i].OnTimePercentage
+			sort.Slice(stats, func(i, j int) bool {
+				return stats[j].OnTimePercentage < stats[i].OnTimePercentage
 			})
 
-			return statsRows, nil
+			return stats, nil
 		},
 	)
 }
-
-func airlineFlightInfo(ctx context.Context, db *sql.DB, origin, dest string) (map[string]*airlineStats, error) {
-	rows, err := db.QueryContext(ctx,
-		`SELECT
-			carriers.code, carriers.name, total_flights, last_flight
-		FROM (
-			SELECT
-				carrier, count(*) AS total_flights, max(date) AS last_flight
-			FROM
-				flights
-			WHERE
-				origin=? AND
-				destination=?
-			GROUP BY carrier
-		) AS _
-		INNER JOIN carriers ON carriers.code=carrier
-		`,
-		origin, dest)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	stats := map[string]*airlineStats{}
-
-	for rows.Next() {
-		var (
-			airline  string
-			rowStats airlineStats
-		)
-
-		err := rows.Scan(&airline, &rowStats.Airline, &rowStats.TotalFlights, &rowStats.LastFlight)
-		if err != nil {
-			return nil, err
-		}
-
-		stats[airline] = &rowStats
-	}
-
-	return stats, nil
-}
-
-func delaysByAirline(ctx context.Context, db *sql.DB, origin, dest string) (map[string]int, error) {
-	rows, err := db.QueryContext(ctx,
-		`SELECT
-			carrier, count(*)
-		FROM
-			flights
-		WHERE
-			origin=? AND
-			destination=? AND
-			scheduled_departure_time <= departure_time AND
-			scheduled_arrival_time <= arrival_time
-		GROUP BY carrier`,
-		origin, dest)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	byAirline := map[string]int{}
-
-	for rows.Next() {
-		var (
-			airline string
-			count   int
-		)
-
-		err := rows.Scan(&airline, &count)
-		if err != nil {
-			return nil, err
-		}
-
-		byAirline[airline] = count
-	}
-
-	return byAirline, nil
-}

backendB/app/graphql/graphql.go

@@ -81,28 +81,28 @@ func instrumentResolver(name string, fn graphql.FieldResolveFn) graphql.FieldRes
 		Subsystem: name,
 		Name:      "requests",
 	})
-	prometheus.MustRegister(requests)
+	promRegister(requests)
 
 	errors := prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: "graphql",
 		Subsystem: name,
 		Name:      "errors",
 	})
-	prometheus.MustRegister(errors)
+	promRegister(errors)
 
 	inflight := prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: "graphql",
 		Subsystem: name,
 		Name:      "inflight",
 	})
-	prometheus.MustRegister(inflight)
+	promRegister(inflight)
 
 	responseTime := prometheus.NewHistogram(prometheus.HistogramOpts{
 		Namespace: "graphql",
 		Subsystem: name,
 		Name:      "response_time",
 	})
-	prometheus.MustRegister(responseTime)
+	promRegister(responseTime)
 
 	return func(p graphql.ResolveParams) (r interface{}, err error) {
 		timer := prometheus.NewTimer(responseTime)
@@ -119,3 +119,8 @@ func instrumentResolver(name string, fn graphql.FieldResolveFn) graphql.FieldRes
 		return fn(p)
 	}
 }
+
+func promRegister(c prometheus.Collector) {
+	prometheus.Unregister(c)
+	prometheus.MustRegister(c)
+}

backendB/app/http/http_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/pboyd/flightranker-backend/backendb/app"
@@ -41,7 +42,7 @@ func TestHandler(t *testing.T) {
 		r := httptest.NewRequest("GET", fmt.Sprintf("/?q=%s", c.query), nil)
 		h.ServeHTTP(w, r)
 
-		actual := w.Body.String()
+		actual := strings.TrimSpace(w.Body.String())
 
 		if actual != c.expected {
 			t.Errorf("\ngot:  %s\nwant: %s", actual, c.expected)

backendB/app/mysql/airport_test.go

@@ -55,7 +55,6 @@ func TestAirportSearch(t *testing.T) {
 		{
 			term: "jack",
 			expectedCodes: []string{
-				"CEC",
 				"JAC",
 				"JAN",
 				"JAX",
@@ -82,8 +81,13 @@ func TestAirportSearch(t *testing.T) {
 		}
 
 		for i := range c.expectedCodes {
-			if actual[i].Code != c.expectedCodes[i] {
-				t.Errorf("%q-%d: got %q, want %q", c.term, i, actual[i].Code, c.expectedCodes[i])
+			var actualCode string
+			if i < len(actual) {
+				actualCode = actual[i].Code
+			}
+
+			if actualCode != c.expectedCodes[i] {
+				t.Errorf("%q-%d: got %q, want %q", c.term, i, actualCode, c.expectedCodes[i])
 			}
 		}
 	}

backendB/app/mysql/flight_stats.go

@@ -13,65 +13,48 @@ func (s *Store) FlightStatsByAirline(ctx context.Context, origin, dest string) (
 		return nil, err
 	}
 
-	delays, err := s.delaysByAirline(ctx, origin, dest)
-	if err != nil {
-		return nil, err
-	}
-
-	for code := range stats {
-		stats[code].TotalDelays = delays[code]
-	}
-
-	statsRows := make([]*app.FlightStats, len(stats))
-	i := 0
-	for code := range stats {
-		statsRows[i] = stats[code]
-		i++
-	}
-
-	sort.Slice(statsRows, func(i, j int) bool {
-		return statsRows[j].OnTimePercentage() < statsRows[i].OnTimePercentage()
+	sort.Slice(stats, func(i, j int) bool {
+		return stats[j].OnTimePercentage() < stats[i].OnTimePercentage()
 	})
 
-	return statsRows, nil
+	return stats, nil
 }
 
-func (s *Store) airlineFlightInfo(ctx context.Context, origin, dest string) (map[string]*app.FlightStats, error) {
+func (s *Store) airlineFlightInfo(ctx context.Context, origin, dest string) ([]*app.FlightStats, error) {
 	rows, err := s.db.QueryContext(ctx,
 		`SELECT
-			carriers.code, carriers.name, total_flights, last_flight
-		FROM (
-			SELECT
-				carrier, count(*) AS total_flights, max(date) AS last_flight
-			FROM
-				flights
-			WHERE
-				origin=? AND
-				destination=?
-			GROUP BY carrier
-		) AS _
-		INNER JOIN carriers ON carriers.code=carrier
+			carriers.name AS carrier_name, total_flights, delays_flights, last_flight
+		FROM
+			(
+				SELECT
+					carrier AS carrier_code,
+					SUM(total_flights) AS total_flights,
+					SUM(delayed_flights) AS delays_flights,
+					MAX(date) AS last_flight
+				FROM
+					flights_day
+				WHERE origin=? AND destination=?
+				GROUP BY carrier_code
+			) AS stats
+		INNER JOIN carriers ON carrier_code=carriers.code
 		`,
 		origin, dest)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
 
-	stats := map[string]*app.FlightStats{}
+	stats := []*app.FlightStats{}
 
 	for rows.Next() {
-		var (
-			airline  string
-			rowStats app.FlightStats
-		)
+		var row app.FlightStats
 
-		err := rows.Scan(&airline, &rowStats.Airline, &rowStats.TotalFlights, &rowStats.LastFlight)
+		err := rows.Scan(&row.Airline, &row.TotalFlights, &row.TotalDelays, &row.LastFlight)
 		if err != nil {
 			return nil, err
 		}
 
-		stats[airline] = &rowStats
+		stats = append(stats, &row)
 	}
 
 	return stats, nil

backendB/app/mysql/flight_stats_test.go

@@ -18,9 +18,7 @@ func TestFlightStatsByAirline(t *testing.T) {
 			origin: "DEN",
 			dest:   "LAS",
 			expected: []*app.FlightStats{
-				{Airline: "ExpressJet Airlines Inc."},
 				{Airline: "Frontier Airlines Inc."},
-				{Airline: "SkyWest Airlines Inc."},
 				{Airline: "Southwest Airlines Co."},
 				{Airline: "Spirit Air Lines"},
 				{Airline: "United Air Lines Inc."},
@@ -46,6 +44,11 @@ func TestFlightStatsByAirline(t *testing.T) {
 		})
 
 		for i := range c.expected {
+			if i >= len(actual) {
+				t.Errorf("%s-%s-%d: missing item", c.origin, c.dest, i)
+				continue
+			}
+
 			if actual[i].Airline != c.expected[i].Airline {
 				t.Errorf("%s-%s-%d: got Airline %q, want %q", c.origin, c.dest, i, actual[i].Airline, c.expected[i].Airline)
 			}

backendB/go.mod

@@ -3,6 +3,8 @@ module github.com/pboyd/flightranker-backend/backendb
 require (
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/graphql-go/graphql v0.7.8
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/pboyd/flightranker-backend/backendtest v0.0.0
 	github.com/prometheus/client_golang v1.1.0
 	google.golang.org/appengine v1.6.2 // indirect

sql/00_schema.sql

@@ -63,3 +63,23 @@ CREATE TABLE flights (
     INDEX destination_idx (destination),
     INDEX date_idx (date)
 );
+
+CREATE TABLE flights_day (
+    date DATE NOT NULL,
+    carrier VARCHAR(6),
+    origin CHAR(3),
+    destination CHAR(3),
+
+    total_flights SMALLINT,
+    delayed_flights SMALLINT,
+
+    PRIMARY KEY (date, carrier, origin, destination),
+    FOREIGN KEY (carrier) REFERENCES carriers(code),
+    FOREIGN KEY (origin) REFERENCES airports(code),
+    FOREIGN KEY (destination) REFERENCES airports(code),
+
+    INDEX carrier_idx (carrier),
+    INDEX origin_idx (origin),
+    INDEX destination_idx (destination),
+    INDEX date_idx (date)
+);