Skip to content

Prune #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
midwire merged 3 commits into from
Apr 20, 2018
Merged

Prune #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1b04543
remove midwire_common dependency
firien Jan 3, 2018
14a9bf1
writing to in-memory db is roughly 3x faster
firien Jan 3, 2018
39cb44b
moved timer to include db write
firien Jan 4, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions Gemfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
source 'http://rubygems.org'
source 'https://rubygems.org'

gem 'midwire_common'
Copy link
Copy Markdown
Collaborator

@cmckni3 cmckni3 Jan 4, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why was midwire_common removed? I believe it contains some rake tasks for the release process.

Copy link
Copy Markdown
Author

@firien firien Jan 4, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

eh - I always like as few dependencies as possible. On the master branch bundle exec rake --tasks only lists those found in lib/tasks/data.rake

But I can include it again if you wish.

gem 'pry-nav'
gem 'rake'
gem 'rubyzip', '~> 1.2.1'
gem 'sqlite3'
18 changes: 2 additions & 16 deletions Gemfile.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,17 @@
GEM
remote: http://rubygems.org/
remote: https://rubygems.org/
specs:
coderay (1.1.2)
method_source (0.8.2)
midwire_common (0.1.16)
thor (~> 0.19)
pry (0.10.4)
coderay (~> 1.1.0)
method_source (~> 0.8.1)
slop (~> 3.4)
pry-nav (0.2.4)
pry (>= 0.9.10, < 0.11.0)
rake (12.2.1)
rubyzip (1.2.1)
slop (3.6.0)
sqlite3 (1.3.11)
thor (0.19.1)

PLATFORMS
ruby

DEPENDENCIES
midwire_common
pry-nav
rake
rubyzip (~> 1.2.1)
sqlite3

BUNDLED WITH
1.15.4
1.16.0
175 changes: 89 additions & 86 deletions lib/tasks/data.rake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ require 'csv'
require 'open-uri'
require 'zip'
require 'yaml'
require 'midwire_common/string'
# require 'pry'

# rubocop:disable Metrics/BlockLength
namespace :data do
Expand Down Expand Up @@ -87,75 +85,75 @@ namespace :data do

# desc 'Create countries table'
task create_countries_table: :kill_db do
schema = <<-STOP.here_with_pipe
|create table countries (
| id integer not null primary key,
| alpha2 varchar(2) not null,
| alpha3 varchar(3),
| iso varchar(3),
| name varchar(255) not null
|)
STOP
schema = <<-SQL
CREATE TABLE countries (
id INTEGER NOT NULL PRIMARY KEY,
alpha2 VARCHAR(2) NOT NULL,
alpha3 VARCHAR(3),
iso VARCHAR(3),
name VARCHAR(255) NOT NULL
)
SQL
database.execute_batch(schema)
ndx = <<-STOP.here_with_pipe(' ')
|CREATE UNIQUE INDEX "main"."unique_country_alpha2"
|ON countries (alpha2 COLLATE NOCASE ASC);
STOP
ndx = <<-SQL
CREATE UNIQUE INDEX main.unique_country_alpha2
ON countries (alpha2 COLLATE NOCASE ASC);
SQL
database.execute_batch(ndx)
end

# desc 'Create states table'
task create_states_table: :create_countries_table do
schema = <<-STOP.here_with_pipe
|create table states (
| id integer not null primary key,
| country_id integer not null,
| abbr varchar(2) not null,
| name varchar(255)
|)
STOP
schema = <<-SQL
CREATE TABLE states (
id INTEGER NOT NULL PRIMARY KEY,
country_id INTEGER NOT NULL,
abbr VARCHAR(2) NOT NULL,
name VARCHAR(255)
)
SQL
database.execute_batch(schema)
ndx = <<-STOP.here_with_pipe(' ')
|CREATE UNIQUE INDEX "main"."unique_state"
|ON states (abbr, country_id COLLATE NOCASE ASC);
STOP
ndx = <<-SQL
CREATE UNIQUE INDEX main.unique_state
ON states (abbr, country_id COLLATE NOCASE ASC);
SQL
database.execute_batch(ndx)
end

# desc 'Create counties table'
task create_counties_table: :create_states_table do
schema = <<-STOP.here_with_pipe
|create table counties (
| id integer not null primary key,
| state_id integer,
| abbr varchar(255),
| name varchar(255),
| county_seat varchar(255)
|)
STOP
schema = <<-SQL
CREATE TABLE counties (
id INTEGER NOT NULL PRIMARY KEY,
state_id INTEGER,
abbr VARCHAR(255),
name VARCHAR(255),
county_seat VARCHAR(255)
)
SQL
database.execute_batch(schema)
end

# desc 'Create zipcodes table'
task create_zipcodes_table: :create_counties_table do
schema = <<-STOP.here_with_pipe
|create table zipcodes (
| id integer not null primary key,
| code varchar(10) not null,
| state_id integer,
| county_id integer,
| city varchar(255),
| area_code varchar(3),
| lat float,
| lon float,
| accuracy varchar(8)
|)
STOP
schema = <<-SQL
CREATE TABLE zipcodes (
id INTEGER NOT NULL PRIMARY KEY,
code VARCHAR(10) NOT NULL,
state_id INTEGER,
county_id INTEGER,
city VARCHAR(255),
area_code VARCHAR(3),
lat FLOAT,
lon FLOAT,
accuracy VARCHAR(8)
)
SQL
database.execute_batch(schema)
ndx = <<-STOP.here_with_pipe(' ')
|CREATE UNIQUE INDEX "main"."unique_zipcode"
|ON zipcodes (state_id, code, city COLLATE NOCASE ASC);
STOP
ndx = <<-SQL
CREATE UNIQUE INDEX main.unique_zipcode
ON zipcodes (state_id, code, city COLLATE NOCASE ASC);
SQL
database.execute_batch(ndx)
end

Expand All @@ -181,13 +179,13 @@ namespace :data do
count += 1

# insert country
sql = <<-STOP.here_with_pipe(' ')
|INSERT INTO countries (alpha2, alpha3, iso, name)
|VALUES ('#{row['COUNTRY']}',
| '#{country_hash[:alpha3]}',
| '#{country_hash[:iso]}',
| '#{country_hash[:name]}')
STOP
sql = <<-SQL
INSERT INTO countries (alpha2, alpha3, iso, name)
VALUES ('#{row['COUNTRY']}',
'#{country_hash[:alpha3]}',
'#{country_hash[:iso]}',
'#{country_hash[:name]}')
SQL
begin
database.execute(sql)
rescue SQLite3::ConstraintException
Expand All @@ -197,13 +195,13 @@ namespace :data do
# state
if row['STATE']
country_id = get_country_id(row['COUNTRY'])
sql = <<-STOP.here_with_pipe(' ')
|INSERT INTO states (abbr, name, country_id)
|VALUES ('#{row['SHORT_STATE']}',
| '#{escape_single_quotes(row['STATE'])}',
| #{country_id}
|)
STOP
sql = <<-SQL
INSERT INTO states (abbr, name, country_id)
VALUES ('#{row['SHORT_STATE']}',
'#{escape_single_quotes(row['STATE'])}',
#{country_id}
)
SQL
begin
database.execute(sql)
rescue StandardError => err
Expand All @@ -214,13 +212,13 @@ namespace :data do
# county
if row['COUNTY']
state_id = get_state_id(row['SHORT_STATE'])
sql = <<-STOP.here_with_pipe(' ')
|INSERT INTO counties (state_id, abbr, name)
|VALUES ('#{state_id}',
| '#{row['SHORT_COUNTY']}',
| '#{escape_single_quotes(row['COUNTY'])}'
|)
STOP
sql = <<-SQL
INSERT INTO counties (state_id, abbr, name)
VALUES ('#{state_id}',
'#{row['SHORT_COUNTY']}',
'#{escape_single_quotes(row['COUNTY'])}'
)
SQL
begin
database.execute(sql)
rescue StandardError => err
Expand All @@ -233,17 +231,17 @@ namespace :data do
state_id = get_state_id(row['SHORT_STATE'])
county_id = get_county_id(row['COUNTY'])
city_name = escape_single_quotes(row['CITY'])
sql = <<-STOP.here_with_pipe(' ')
|INSERT INTO zipcodes (code, state_id, county_id, city, lat, lon, accuracy)
|VALUES ('#{row['POSTAL_CODE']}',
| '#{state_id}',
| '#{county_id}',
| '#{city_name}',
| '#{row['LATITUDE']}',
| '#{row['LONGITUDE']}',
| '#{row['ACCURACY']}'
|)
STOP
sql = <<-SQL
INSERT INTO zipcodes (code, state_id, county_id, city, lat, lon, accuracy)
VALUES ('#{row['POSTAL_CODE']}',
'#{state_id}',
'#{county_id}',
'#{city_name}',
'#{row['LATITUDE']}',
'#{row['LONGITUDE']}',
'#{row['ACCURACY']}'
)
SQL
begin
database.execute(sql)
rescue SQLite3::ConstraintException => err
Expand All @@ -256,14 +254,19 @@ namespace :data do
end
end

# save to disk
disk_db = SQLite3::Database.new('free_zipcode_data.sqlite3')
backup = SQLite3::Backup.new(disk_db, 'main', database, 'main')
backup.step(-1)
backup.finish
end_time = Time.now - start_time
puts ">>>> Completed in #{end_time} seconds"
end

private

def database
@db ||= SQLite3::Database.new('free_zipcode_data.sqlite3')
@db ||= SQLite3::Database.new(':memory:')
end

def get_country_id(country)
Expand Down