Skip to content
Permalink
Browse files

Roll up flights by day

  • Loading branch information...
pboyd committed Sep 11, 2019
1 parent 6d21281 commit cd81b2bc591bd53578dde54059bd12f230dd35ec
@@ -34,7 +34,11 @@ following months:
* [February 2019](https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2019_2.zip)
* [March 2019](https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2019_3.zip)

After the data has been loaded, run `sql/updates/mark_active_airports.sql`.
After the data has been loaded, run the SQL files in `sql/updates`:

```sh
cat sql/updates/*.sql | mysql -uflightdb -pflightdb -h 127.0.0.1 flightdb
```

## Configuration

@@ -1,7 +1,6 @@
package main

import (
"context"
"database/sql"
"sort"
"strings"
@@ -42,110 +41,51 @@ func resolveFlightStatsByAirline(db *sql.DB) graphql.FieldResolveFn {
return nil, nil
}

stats, err := airlineFlightInfo(p.Context, db, origin, dest)
rows, err := db.QueryContext(p.Context,
`SELECT
carriers.name AS carrier_name, total_flights, delays_flights, last_flight
FROM
(
SELECT
carrier AS carrier_code,
SUM(total_flights) AS total_flights,
SUM(delayed_flights) AS delays_flights,
MAX(date) AS last_flight
FROM
flights_day
WHERE origin=? AND destination=?
GROUP BY carrier_code
) AS stats
INNER JOIN carriers ON carrier_code=carriers.code
`,
origin, dest)
if err != nil {
return nil, err
}
defer rows.Close()

delays, err := delaysByAirline(p.Context, db, origin, dest)
if err != nil {
return nil, err
}
stats := []*airlineStats{}

for code := range stats {
stats[code].OnTimePercentage = (1.0 - float64(delays[code])/float64(stats[code].TotalFlights)) * 100
}
for rows.Next() {
var (
row airlineStats
delayedFlights int
)
err := rows.Scan(&row.Airline, &row.TotalFlights, &delayedFlights, &row.LastFlight)
if err != nil {
return nil, err
}

row.OnTimePercentage = (1.0 - float64(delayedFlights)/float64(row.TotalFlights)) * 100

statsRows := make([]*airlineStats, len(stats))
i := 0
for code := range stats {
statsRows[i] = stats[code]
i++
stats = append(stats, &row)
}

sort.Slice(statsRows, func(i, j int) bool {
return statsRows[j].OnTimePercentage < statsRows[i].OnTimePercentage
sort.Slice(stats, func(i, j int) bool {
return stats[j].OnTimePercentage < stats[i].OnTimePercentage
})

return statsRows, nil
return stats, nil
},
)
}

func airlineFlightInfo(ctx context.Context, db *sql.DB, origin, dest string) (map[string]*airlineStats, error) {
rows, err := db.QueryContext(ctx,
`SELECT
carriers.code, carriers.name, total_flights, last_flight
FROM (
SELECT
carrier, count(*) AS total_flights, max(date) AS last_flight
FROM
flights
WHERE
origin=? AND
destination=?
GROUP BY carrier
) AS _
INNER JOIN carriers ON carriers.code=carrier
`,
origin, dest)
if err != nil {
return nil, err
}
defer rows.Close()

stats := map[string]*airlineStats{}

for rows.Next() {
var (
airline string
rowStats airlineStats
)

err := rows.Scan(&airline, &rowStats.Airline, &rowStats.TotalFlights, &rowStats.LastFlight)
if err != nil {
return nil, err
}

stats[airline] = &rowStats
}

return stats, nil
}

func delaysByAirline(ctx context.Context, db *sql.DB, origin, dest string) (map[string]int, error) {
rows, err := db.QueryContext(ctx,
`SELECT
carrier, count(*)
FROM
flights
WHERE
origin=? AND
destination=? AND
scheduled_departure_time <= departure_time AND
scheduled_arrival_time <= arrival_time
GROUP BY carrier`,
origin, dest)
if err != nil {
return nil, err
}
defer rows.Close()

byAirline := map[string]int{}

for rows.Next() {
var (
airline string
count int
)

err := rows.Scan(&airline, &count)
if err != nil {
return nil, err
}

byAirline[airline] = count
}

return byAirline, nil
}
@@ -81,28 +81,28 @@ func instrumentResolver(name string, fn graphql.FieldResolveFn) graphql.FieldRes
Subsystem: name,
Name: "requests",
})
prometheus.MustRegister(requests)
promRegister(requests)

errors := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "graphql",
Subsystem: name,
Name: "errors",
})
prometheus.MustRegister(errors)
promRegister(errors)

inflight := prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "graphql",
Subsystem: name,
Name: "inflight",
})
prometheus.MustRegister(inflight)
promRegister(inflight)

responseTime := prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "graphql",
Subsystem: name,
Name: "response_time",
})
prometheus.MustRegister(responseTime)
promRegister(responseTime)

return func(p graphql.ResolveParams) (r interface{}, err error) {
timer := prometheus.NewTimer(responseTime)
@@ -119,3 +119,8 @@ func instrumentResolver(name string, fn graphql.FieldResolveFn) graphql.FieldRes
return fn(p)
}
}

func promRegister(c prometheus.Collector) {
prometheus.Unregister(c)
prometheus.MustRegister(c)
}
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net/http/httptest"
"strings"
"testing"

"github.com/pboyd/flightranker-backend/backendb/app"
@@ -41,7 +42,7 @@ func TestHandler(t *testing.T) {
r := httptest.NewRequest("GET", fmt.Sprintf("/?q=%s", c.query), nil)
h.ServeHTTP(w, r)

actual := w.Body.String()
actual := strings.TrimSpace(w.Body.String())

if actual != c.expected {
t.Errorf("\ngot: %s\nwant: %s", actual, c.expected)
@@ -55,7 +55,6 @@ func TestAirportSearch(t *testing.T) {
{
term: "jack",
expectedCodes: []string{
"CEC",
"JAC",
"JAN",
"JAX",
@@ -82,8 +81,13 @@ func TestAirportSearch(t *testing.T) {
}

for i := range c.expectedCodes {
if actual[i].Code != c.expectedCodes[i] {
t.Errorf("%q-%d: got %q, want %q", c.term, i, actual[i].Code, c.expectedCodes[i])
var actualCode string
if i < len(actual) {
actualCode = actual[i].Code
}

if actualCode != c.expectedCodes[i] {
t.Errorf("%q-%d: got %q, want %q", c.term, i, actualCode, c.expectedCodes[i])
}
}
}
@@ -13,65 +13,48 @@ func (s *Store) FlightStatsByAirline(ctx context.Context, origin, dest string) (
return nil, err
}

delays, err := s.delaysByAirline(ctx, origin, dest)
if err != nil {
return nil, err
}

for code := range stats {
stats[code].TotalDelays = delays[code]
}

statsRows := make([]*app.FlightStats, len(stats))
i := 0
for code := range stats {
statsRows[i] = stats[code]
i++
}

sort.Slice(statsRows, func(i, j int) bool {
return statsRows[j].OnTimePercentage() < statsRows[i].OnTimePercentage()
sort.Slice(stats, func(i, j int) bool {
return stats[j].OnTimePercentage() < stats[i].OnTimePercentage()
})

return statsRows, nil
return stats, nil
}

func (s *Store) airlineFlightInfo(ctx context.Context, origin, dest string) (map[string]*app.FlightStats, error) {
func (s *Store) airlineFlightInfo(ctx context.Context, origin, dest string) ([]*app.FlightStats, error) {
rows, err := s.db.QueryContext(ctx,
`SELECT
carriers.code, carriers.name, total_flights, last_flight
FROM (
SELECT
carrier, count(*) AS total_flights, max(date) AS last_flight
FROM
flights
WHERE
origin=? AND
destination=?
GROUP BY carrier
) AS _
INNER JOIN carriers ON carriers.code=carrier
carriers.name AS carrier_name, total_flights, delays_flights, last_flight
FROM
(
SELECT
carrier AS carrier_code,
SUM(total_flights) AS total_flights,
SUM(delayed_flights) AS delays_flights,
MAX(date) AS last_flight
FROM
flights_day
WHERE origin=? AND destination=?
GROUP BY carrier_code
) AS stats
INNER JOIN carriers ON carrier_code=carriers.code
`,
origin, dest)
if err != nil {
return nil, err
}
defer rows.Close()

stats := map[string]*app.FlightStats{}
stats := []*app.FlightStats{}

for rows.Next() {
var (
airline string
rowStats app.FlightStats
)
var row app.FlightStats

err := rows.Scan(&airline, &rowStats.Airline, &rowStats.TotalFlights, &rowStats.LastFlight)
err := rows.Scan(&row.Airline, &row.TotalFlights, &row.TotalDelays, &row.LastFlight)
if err != nil {
return nil, err
}

stats[airline] = &rowStats
stats = append(stats, &row)
}

return stats, nil
@@ -18,9 +18,7 @@ func TestFlightStatsByAirline(t *testing.T) {
origin: "DEN",
dest: "LAS",
expected: []*app.FlightStats{
{Airline: "ExpressJet Airlines Inc."},
{Airline: "Frontier Airlines Inc."},
{Airline: "SkyWest Airlines Inc."},
{Airline: "Southwest Airlines Co."},
{Airline: "Spirit Air Lines"},
{Airline: "United Air Lines Inc."},
@@ -46,6 +44,11 @@ func TestFlightStatsByAirline(t *testing.T) {
})

for i := range c.expected {
if i >= len(actual) {
t.Errorf("%s-%s-%d: missing item", c.origin, c.dest, i)
continue
}

if actual[i].Airline != c.expected[i].Airline {
t.Errorf("%s-%s-%d: got Airline %q, want %q", c.origin, c.dest, i, actual[i].Airline, c.expected[i].Airline)
}
@@ -3,6 +3,8 @@ module github.com/pboyd/flightranker-backend/backendb
require (
github.com/go-sql-driver/mysql v1.4.1
github.com/graphql-go/graphql v0.7.8
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pboyd/flightranker-backend/backendtest v0.0.0
github.com/prometheus/client_golang v1.1.0
google.golang.org/appengine v1.6.2 // indirect
@@ -63,3 +63,23 @@ CREATE TABLE flights (
INDEX destination_idx (destination),
INDEX date_idx (date)
);

CREATE TABLE flights_day (
date DATE NOT NULL,
carrier VARCHAR(6),
origin CHAR(3),
destination CHAR(3),

total_flights SMALLINT,
delayed_flights SMALLINT,

PRIMARY KEY (date, carrier, origin, destination),
FOREIGN KEY (carrier) REFERENCES carriers(code),
FOREIGN KEY (origin) REFERENCES airports(code),
FOREIGN KEY (destination) REFERENCES airports(code),

INDEX carrier_idx (carrier),
INDEX origin_idx (origin),
INDEX destination_idx (destination),
INDEX date_idx (date)
);

0 comments on commit cd81b2b

Please sign in to comment.
You can’t perform that action at this time.