Skip to content
Permalink
Browse files

Working on test data

  • Loading branch information...
ryanrolds committed May 16, 2019
1 parent 6d08a6a commit f6294c10a5f6fe6661d7cc7e6363f47fbeddd4a5
@@ -301,9 +301,3 @@ DEPENDENCIES
unicorn
web-console (~> 2.0)
whenever

RUBY VERSION
ruby 2.3.3p222

BUNDLED WITH
1.14.6
@@ -136,6 +136,12 @@ $ docker-compose exec mysql mysqldump --no-create-info -u suyc -psuyc suyc censu
$ docker-compose exec mysql mysqldump --no-create-info -u suyc -psuyc suyc zip_boundaries > data/zip_codes.sql
```

### Creating test data

After loading boundaries and submissions you can distribute the submissions across all Zip Codes and Census Tracts by running:

$ docker-compose run frontend rake create_test_data

# Governance and contribution

See [CONTRIBUTING.md](CONTRIBUTING.md).
@@ -49,7 +49,9 @@ def self.create_submissions(data, test_type)

def self.import
client = bigquery_init
zip_codes = "'#{Submission::ZIP_CODES.join("','")}'"

zips = ZipBoundary.pluck(:name)
zip_codes = "'#{zips.join("','")}'"

upload_query = upload_query(zip_codes)
download_query = download_query(zip_codes)
@@ -32,20 +32,15 @@ class Submission < ActiveRecord::Base
}

CSV_COLUMNS = [
'Response #', 'Source', 'Day', 'Time', 'How Are You Testing', 'Zip', 'Census Tract', 'Provider', 'How are you connected', 'Price Per Month', 'Advertised Download Speed', 'Satisfaction Rating', 'Download Speed', 'Upload Speed', 'Advertised Price Per Mbps', 'Actual Price Per Mbps', 'Ping'
'Response #', 'Source', 'Day', 'Time', 'How Are You Testing', 'Zip', 'Census Tract',
'Provider', 'How are you connected', 'Price Per Month', 'Advertised Download Speed',
'Satisfaction Rating', 'Download Speed', 'Upload Speed', 'Advertised Price Per Mbps',
'Actual Price Per Mbps', 'Ping'
]

ZIP_CODES = [
'97405', '97401', '97424','97403','97439','97463','97438','97493',
'97452','97477','97404','97426','97487','97408','97448','97478',
'97461','97437','97451','97434','97455','97413','97419','97490',
'97412','97488','97431','97480','97430','97453','97489','97454',
'97492','97402'
]

GEOS_IDS = ['41039003700','41039001202','41039000402','41039001400','41039003201','41039003202','41039001500','41039001001','41039001302','41039002403','41039002404','41039003800','41039005200','41039003000','41039003900','41039000708','41039003600','41039004700','41039002800','41039002401','41039000500','41039002904','41039001101','41039001600','41039002902','41039001904','41039003301','41039001801','41039005000','41039001804','41039004501','41039003302','41039005300','41039000100','41039002302','41039002600','41039002002','41039003101','41039004200','41039000706','41039002001','41039000705','41039001902','41039000702','41039001903','41039004100','41039001700','41039002503','41039002201','41039000300','41039001102','41039002202','41039005100','41039003102','41039000200','41039000902','41039002102','41039004800','41039004404','41039000403','41039003400','41039000707','41039002501','41039003500','41039002101','41039004900','41039002301','41039004000','41039002903','41039001803','41039002700','41039001002','41039004502','41039004300','41039002504','41039000800','41039004403','41039004600','41039004401','41039005400','41039000904','41039004405','41039000404','41039001201','41039001301','41039000903']

CENSUS_CODES = ['3700','1202','402 ','1400','3201','3202','1500','1001','1302','2403','2404','3800','5200','3000','3900','708 ','3600','4700','2800','2401','500 ','2904','1101','1600','2902','1904','3301','1801','5000','1804','4501','3302','5300','100 ','2302','2600','2002','3101','4200','706 ','2001','705 ','1902','702 ','1903','4100','1700','2503','2201','300 ','1102','2202','5100','3102','200 ','902 ','2102','4800','4404','403 ','3400','707 ','2501','3500','2101','4900','2301','4000','2903','1803','2700','1002','4502','4300','2504','800 ','4403','4600','4401','5400','904 ','4405','404 ','1201','1301','903']
ZIP_CODES = ZipBoundary.pluck(:name)
CENSUS_CODES = CensusBoundary.pluck(:name)
GEOS_IDS = CensusBoundary.pluck(:geo_id)

validates :testing_for, length: { maximum: 20 }
validates :provider, :connected_with, length: { maximum: 50 }
@@ -94,11 +89,7 @@ def self.create_submission(params)
end

def valid_attributes?
has_required_fields? && valid_zip_code?
end

def valid_zip_code?
ZIP_CODES.include?(zip_code)
has_required_fields?
end

def has_required_fields?
@@ -146,7 +137,7 @@ def self.set_mapbox_polygon_data(params, data=[])
params[:census_code] = CENSUS_CODES if params[:census_code] == ['all']

polygon_data = valid_test
polygon_data = polygon_data.where(provider: providers)
#polygon_data = polygon_data.where(provider: providers)
polygon_data = polygon_data.with_date_range(start_date, end_date) if date_range.present?
polygon_data = polygon_data.with_zip_code(params[:zip_code]) if params[:zip_code].present?
polygon_data = polygon_data.with_census_code(params[:census_code]) if params[:census_code].present?
@@ -162,6 +153,8 @@ def self.set_mapbox_polygon_data(params, data=[])

boundaries = zip_boundaries

puts polygon_data.length()

polygon_data.each do |zip_code, submissions|
attribute_name = speed_attribute(params[:test_type])
median_speed = median(submissions.map(&:"#{attribute_name}")).to_f
@@ -206,14 +199,17 @@ def self.set_mapbox_census_data(params, data=[])
params[:census_code] = CENSUS_CODES if params[:census_code] == ['all']

polygon_data = valid_test
polygon_data = polygon_data.where(provider: providers)
#polygon_data = polygon_data.where(provider: providers)
polygon_data = polygon_data.with_date_range(start_date, end_date) if date_range.present?
polygon_data = polygon_data.mapbox_filter_by_census_code(params[:test_type]) if params[:test_type].present?

#boundaries = Rails.cache.fetch('census_boundaries', expires_in: 2.hours) do
census_boundaries = {}
CensusBoundary.where(geo_id: GEOS_IDS).each do |boundary|
census_boundaries[boundary.name.to_i.to_s] = { bounds: boundary.bounds }
census_boundaries[boundary.name.to_i.to_s] = {
geom_type: boundary.geom_type,
bounds: boundary.bounds
}
end
#census_boundaries
#end
@@ -225,11 +221,10 @@ def self.set_mapbox_census_data(params, data=[])
median_speed = median(submissions.map(&:"#{attribute_name}")).to_f
census_boundary = boundaries[census_code.to_s]

if census_boundary.present?
census_coordinates = census_boundary[:bounds]
else
census_coordinates = CensusBoundary.first.bounds
end
next if census_boundary.present? == false

census_coordinates = census_boundary[:bounds]
geom_type = census_boundary[:geom_type]

feature = {
'type': 'Feature',
@@ -245,7 +240,7 @@ def self.set_mapbox_census_data(params, data=[])
'color': params['type'] == 'stats' && set_stats_color(submissions.count) || set_color(median_speed),
},
'geometry': {
'type': 'Polygon',
'type': geom_type,
'coordinates': census_coordinates,
}
}
@@ -29,8 +29,8 @@
<meta name="robots" content="noindex,nofollow">
<% end %>

<script src='https://api.mapbox.com/mapbox.js/v2.3.0/mapbox.js'></script>
<link href='https://api.mapbox.com/mapbox.js/v2.3.0/mapbox.css' rel='stylesheet' />
<script src='https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.js'></script>
<link href='https://api.mapbox.com/mapbox.js/v2.4.0/mapbox.css' rel='stylesheet' />
<link href='https://fonts.googleapis.com/css?family=Roboto:400,300,500,700' rel='stylesheet' type='text/css'>

</head>
@@ -1,8 +1,8 @@
<script src='https://api.mapbox.com/mapbox.js/plugins/leaflet-markercluster/v0.4.0/leaflet.markercluster.js'></script>
<link href='https://api.mapbox.com/mapbox.js/plugins/leaflet-markercluster/v1.0.0/MarkerCluster.Default.css' rel='stylesheet' />

<script src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.32.1/mapbox-gl.js'></script>
<link href='https://api.tiles.mapbox.com/mapbox-gl-js/v0.32.1/mapbox-gl.css' rel='stylesheet' />
<script src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.54.0/mapbox-gl.js'></script>
<link href='https://api.tiles.mapbox.com/mapbox-gl-js/v0.54.0/mapbox-gl.css' rel='stylesheet' />

<div id='all-results'>
<%= hidden_field_tag :root_url, root_url %>
@@ -13,7 +13,7 @@ services:
build: ../speedupamerica-migrator
env_file:
- local.env
command: ["rake", "db:setup"]
command: ["rake", "db:migrate"]
links:
- mysql
depends_on:
@@ -21,14 +21,19 @@ task :populate_census_tracts => [:environment] do
# if the zip code doesn't include parts of Lane county, ignore it
next if !(data["COUNTYFP"].include? "39")

# if it's already in ZipBoundary, ignore it
# if it's already in CensusBoundary, ignore it
next if CensusBoundary.where(name: data["TRACTCE"]).present?

# get polygon
polygon = GeoRuby::SimpleFeatures::MultiPolygon.from_ewkt(data["tract_polygons"])
geom_type = "Polygon"
polygon = GeoRuby::SimpleFeatures::Polygon.from_ewkt(data["tract_polygons"])
if data["tract_polygons"].start_with?('MULTIPOLYGON')
geom_type = "MultiPolygon"
polygon = GeoRuby::SimpleFeatures::MultiPolygon.from_ewkt(data["tract_polygons"])
end

# otherwise, create a new record
CensusBoundary.create(name: data["TRACTCE"], geo_id: data["GEOID"], bounds: polygon.to_coordinates())
CensusBoundary.create(name: data["TRACTCE"], geo_id: data["GEOID"], geom_type: geom_type,
bounds: polygon.to_coordinates())

# increment the count
add_count += 1
@@ -21,17 +21,16 @@ task :populate_zip_boundaries => [:environment] do
next if data["state_code"] != "OR"

# if the zip code doesn't include parts of Lane county, ignore it
next if !(data["county"].include? "Lane")
#next if !(data["county"].include? "Lane")

# if it's already in ZipBoundary, ignore it
next if ZipBoundary.where(name: data["zip_code"]).present?

# get polygon
polygon = GeoRuby::SimpleFeatures::MultiPolygon.from_ewkt(data["zcta_geom"])


zip_type = "Polygon"
polygon = GeoRuby::SimpleFeatures::Polygon.from_ewkt(data["zcta_geom"])
if data["zcta_geom"].start_with?('MULTIPOLYGON')
zip_type = "MultiPolygon"
polygon = GeoRuby::SimpleFeatures::MultiPolygon.from_ewkt(data["zcta_geom"])
end

# otherwise, create a new record
@@ -45,6 +45,7 @@ task :create_test_data => [:environment] do
submissions = Submission.all()
submissions.each_with_index do |s, index|
s.census_code = tracts[index % countTracts].name
s.census_status = 'saved'
s.zip_code = zips[index % countZips].name
s.save
end

0 comments on commit f6294c1

Please sign in to comment.
You can’t perform that action at this time.