First cut of refactor

.gitignore

@@ -1,8 +1,6 @@
-ZIP_CODES.txt
-zip_codes.zip
-zip_dump.csv
-zip_codes.db
 /.bundle
 /free_zipcode_data.sqlite3
 /data
 /build
+/stubs
+/vendor/bundle/

.rspec

@@ -0,0 +1 @@
+--require spec_helper

.rubocop.yml

@@ -1,4 +1,6 @@
 AllCops:
+  TargetRubyVersion: 2.3
+
   # Include gemspec and Rakefile
   Include:
     - '**/*.gemspec'
@@ -15,6 +17,7 @@ AllCops:
   Exclude:
     - 'vendor/**/*'
     - 'stubs/**/*'
+    - 'spec/support/shared_contexts/*'
 
 # Checks formatting of special comments
 CommentAnnotation:
@@ -49,32 +52,16 @@ Style/RaiseArgs:
 Style/DoubleNegation:
   Enabled: false
 
-Style/SignalException:
-  EnforcedStyle: semantic
-
-Style/ClassAndModuleChildren:
-  Enabled: false
-
-Style/MultilineMethodCallIndentation:
-  EnforcedStyle: indented
-  IndentationWidth: 4
-
-Style/CaseIndentation:
-  Enabled: false
-
-Style/TrivialAccessors:
-  Enabled: false
-
-Style/NumericLiterals:
+Style/PerlBackrefs:
   Enabled: false
 
 ########################################
 # Lint Cops
 
 Lint/Eval:
-  Enabled: true
+  Enabled: false
 
-Lint/AssignmentInCondition:
+Lint/HandleExceptions:
   Enabled: false
 
 ########################################
@@ -85,7 +72,14 @@ Metrics/LineLength:
 
 Metrics/MethodLength:
   CountComments: false # count full line comments?
-  Max: 20
+  Max: 30
 
 Metrics/ClassLength:
   Max: 120
+
+Metrics/AbcSize:
+  Enabled: false
+
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Metrics/MethodLength
+# rubocop:disable Metrics/BlockLength

Gemfile

@@ -1,7 +1,6 @@
-source 'http://rubygems.org'
+# frozen_string_literal: true
 
-gem 'midwire_common'
-gem 'pry-nav'
-gem 'rake'
-gem 'rubyzip', '~> 1.2.1'
-gem 'sqlite3'
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+gemspec

Gemfile.lock

@@ -1,31 +1,79 @@
+PATH
+  remote: .
+  specs:
+    free_zipcode_data (1.0.0)
+      colored (~> 1.2)
+      kiba (~> 2.0)
+      rubyzip (~> 1.2)
+      sqlite3 (~> 1.3)
+      trollop (~> 2.1)
+
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
+    ast (2.4.0)
     coderay (1.1.2)
+    colored (1.2)
+    diff-lcs (1.3)
+    docile (1.3.0)
+    json (2.1.0)
+    kiba (2.0.0)
     method_source (0.8.2)
-    midwire_common (0.1.16)
-      thor (~> 0.19)
+    parallel (1.12.1)
+    parser (2.5.1.0)
+      ast (~> 2.4.0)
+    powerpack (0.1.1)
     pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
     pry-nav (0.2.4)
       pry (>= 0.9.10, < 0.11.0)
+    rainbow (3.0.0)
     rake (12.2.1)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    rubocop (0.55.0)
+      parallel (~> 1.10)
+      parser (>= 2.5)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.9.0)
     rubyzip (1.2.1)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
     slop (3.6.0)
-    sqlite3 (1.3.11)
-    thor (0.19.1)
+    sqlite3 (1.3.13)
+    trollop (2.1.2)
+    unicode-display_width (1.3.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  midwire_common
-  pry-nav
-  rake
-  rubyzip (~> 1.2.1)
-  sqlite3
+  bundler (~> 1.16)
+  free_zipcode_data!
+  pry-nav (~> 0.2)
+  rake (~> 12.0)
+  rspec (~> 3.7)
+  rubocop
+  simplecov
 
 BUNDLED WITH
-   1.15.4
+   1.16.1

README.md

@@ -90,6 +90,10 @@ create table zipcodes (
 
 Both `lat` and `lon`, geocodes, are populated for each zipcode record.
 
+## Development
+
+If you want to run the specs or do development work, set `GEM_ENV='development'`
+
 ## Data License
 
 The zipcode data is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by/3.0/">Creative Commons Attribution 3.0 Unported License</a>, carried forward from [GeoNames](http://www.geonames.org).<br />

bin/free_zipcode_data

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'free_zipcode_data'
+require 'free_zipcode_data/runner'
+
+FreeZipcodeData::Runner.instance.start

country_lookup_table.yml

@@ -10,7 +10,7 @@ AX:
 AL:
   :name: Albania
   :alpha3: ALB
-  :iso: 008
+  :iso: '008'
 DZ:
   :name: Algeria
   :alpha3: DZA

free_zipcode_data.gemspec

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'free_zipcode_data/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'free_zipcode_data'
+  spec.version       = FreeZipcodeData::VERSION
+  spec.authors       = ['Chris Blackburn', 'Chris McKnight']
+  spec.email         = ['87a1779b@opayq.com']
+  spec.summary       = 'Free US postal codes in CSV and SQLite3 format.'
+  spec.description   = spec.summary
+  spec.homepage      = 'https://github.com/midwire/free_zipcode_data'
+  spec.license       = 'MIT'
+
+  spec.required_ruby_version = '>= 2.3.0'
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'pry-nav', '~> 0.2'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.7'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'simplecov'
+
+  spec.add_runtime_dependency 'colored', '~> 1.2'
+  spec.add_runtime_dependency 'kiba', '~> 2.0'
+  spec.add_runtime_dependency 'rubyzip', '~> 1.2'
+  spec.add_runtime_dependency 'sqlite3', '~> 1.3'
+  spec.add_runtime_dependency 'trollop', '~> 2.1'
+end

lib/etl/common.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative 'csv_source'
+require_relative '../free_zipcode_data/country_table'
+require_relative '../free_zipcode_data/state_table'
+require_relative '../free_zipcode_data/county_table'
+require_relative '../free_zipcode_data/zipcode_table'
+
+def show_me
+  transform do |row|
+    ap row
+    row
+  end
+end
+
+def limit(count)
+  count = Integer(count || -1)
+  return if count == -1
+  transform do |row|
+    @counter ||= 0
+    @counter += 1
+    @counter > count ? nil : row
+  end
+end

lib/etl/csv_source.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+class CsvSource
+  attr_reader :filename, :delimeter, :quote_char, :headers
+
+  def initialize(filename:, headers: true, delimeter: "\t", quote_char: '"')
+    @filename   = filename
+    @headers    = headers
+    @delimeter  = delimeter
+    @quote_char = quote_char
+  end
+
+  def each
+    CSV.open(filename,
+      col_sep: delimeter,
+      headers: headers,
+      header_converters: :symbol,
+      quote_char: quote_char) do |csv|
+      csv.each do |row|
+        yield(row.to_hash)
+      end
+    end
+  end
+end

lib/etl/free_zipcode_data_job.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative 'common'
+
+module ETL
+  module FreeZipcodeDataJob
+    module_function
+
+    def setup(country_file, database, logger, options)
+      Kiba.parse do
+        pre_process do
+          logger.info("Processing '#{country_file}' data, please be patient...")
+        end
+
+        source CsvSource, filename: country_file, quote_char: '"', delimeter: ','
+
+        destination FreeZipcodeData::CountryTable,
+          database: database,
+          tablename: options[:country_tablename]
+
+        destination FreeZipcodeData::StateTable,
+          database: database,
+          tablename: options[:state_tablename]
+
+        destination FreeZipcodeData::CountyTable,
+          database: database,
+          tablename: options[:county_tablename]
+
+        destination FreeZipcodeData::ZipcodeTable,
+          database: database,
+          tablename: options[:zipcode_tablename]
+
+      end
+    end
+  end
+end

lib/free_zipcode_data.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'readline'
+
+require 'free_zipcode_data/version'
+
+module FreeZipcodeData
+  def self.root
+    Pathname.new(File.dirname(__FILE__)).parent
+  end
+
+  def self.current_environment
+    ENV.fetch('APP_ENV', 'development')
+  end
+
+  #:nocov:
+  def self.config_file(filename = '.free_zipcode_data.yml')
+    return root.join('spec', 'fixtures', filename) if current_environment == 'test'
+    home = ENV.fetch('HOME')
+    file = ENV.fetch('FZD_CONFIG_FILE', File.join(home, '.free_zipcode_data.yml'))
+    FileUtils.touch(file)
+    file
+  end
+  #:nocov:
+
+  def self.os
+    if RUBY_PLATFORM.match?(/cygwin|mswin|mingw|bccwin|wince|emx/)
+      :retarded
+    else
+      :normal
+    end
+  end
+
+  autoload :CountryTable, 'free_zipcode_data/country_table'
+  autoload :StateTable,   'free_zipcode_data/state_table'
+  autoload :CountyTable,  'free_zipcode_data/county_table'
+  autoload :ZipcodeTable, 'free_zipcode_data/zipcode_table'
+  autoload :DataSource,   'free_zipcode_data/data_source'
+  autoload :Logger,       'free_zipcode_data/logger'
+  autoload :Options,      'free_zipcode_data/options'
+  autoload :Settings,     'free_zipcode_data/settings'
+  autoload :SqliteRam,    'free_zipcode_data/sqlite_ram'
+end