Permalink
Browse files

Remove extractor. This now resides in another project that matches BC…

…C data.
  • Loading branch information...
1 parent 0c32596 commit c7904f4386386cc4aa1c2842f7fb2a37d3e0a5ab @tatey committed Apr 3, 2012
View
@@ -16,8 +16,6 @@ First install.
$ [sudo] gem install translink
-### Scrape
-
Scrape all bus stops, routes and services for Thursday, 24 November 2011 saving
them into a SQLite database named "2011-11-24.sqlite3" in the current working directory.
@@ -27,15 +25,6 @@ Change the path to the SQLite database.
$ translink scrape 2011-11-24 --uri=sqlite:///Users/Tate/Downloads/translink.sqlite3
-### Extract
-
-Extract street1, street2 and the locality of each bus stop from the name and summary. Strips
-white space and numbers.
-
- $ translink extract sqlite:///Users/Tate/Downloads/translink.sqlite3
-
-Not all bus stops have a locality.
-
## Schema
![Class Analysis Diagram](https://github.com/tatey/translink/raw/master/doc/schema.png)
View
@@ -11,7 +11,6 @@
require 'translink/model/route'
require 'translink/model/service'
require 'translink/model/stop'
-require 'translink/model/stop/extractor'
require 'translink/page'
require 'translink/page/route'
require 'translink/page/timetable'
View
@@ -1,6 +1,6 @@
module Translink
class CLI
- RUNNABLE = ['extract', 'help', 'scrape']
+ RUNNABLE = ['help', 'scrape']
attr_accessor :out, :pwd, :__crawler__, :__stop__
@@ -22,16 +22,6 @@ def run line
protected
- def extract input
- return help nil unless input =~ /[A-Za-z]:.+/
- DB.context input do
- __stop__.all.each do |stop|
- stop.extract!
- stop.save!
- end
- end
- end
-
def help input
log 'help'
end
@@ -13,19 +13,9 @@ class Stop
has n, :services
has n, :routes, :through => :services
- attr_accessor :__extractor__
-
def self.find_or_add_from_stop stop
Stop.first_or_create :name => stop.name, :summary => stop.summary
end
-
- def extract!
- __extractor__.new(self).extract!
- end
-
- def __extractor__
- @__extractor__ ||= Extractor
- end
end
end
end
@@ -1,67 +0,0 @@
-module Translink
- module Model
- class Stop::Extractor
- module Nil
- def gsub *args
- self
- end
-
- def strip
- self
- end
- end
-
- REGEXES = {
- /[\s\(]opposite approaching/i => 'opposite_approaching',
- /[\s\(]opposite far side of/i => 'opposite_far_side',
- /[\s\(]opposite/i => 'opposite',
- /[\s\(]approaching/i => 'approaching',
- /[\s\(]far side of/i => 'far_side',
- /[\s\(]at/i => 'at',
- /[\s\(]near/i => 'near'
- }
-
- attr_accessor :stop
-
- def initialize stop
- @stop = stop
- end
-
- def extract!
- stop.street1 = street1
- stop.street2 = street2
- stop.locality = locality
- stop
- end
-
- def street1
- segment = segments.first
- segment =~ /,\s*(.+)/
- ($1 || segment).gsub(/\d-\d|\d/, '').strip
- end
-
- def street2
- segments.last.gsub(/[\(\)]/, '').strip
- end
-
- def locality
- REGEXES[regex]
- end
-
- protected
-
- def regex
- REGEXES.keys.find { |regex| summary_or_name =~ regex }
- end
-
- def segments
- results = summary_or_name.split regex
- results.size == 2 ? results : [summary_or_name, nil.extend(Nil)]
- end
-
- def summary_or_name
- stop.summary ? stop.summary : stop.name
- end
- end
- end
-end
View
@@ -36,20 +36,6 @@ def test_unrecognised_command_executes_help_command
assert_match /help/, @out.string
end
- def test_execute_extract_command_with_invalid_uri
- @cli.run 'extract invalid'
- assert_match /help/, @out.string
- end
-
- def test_execute_extract_command_with_valid_uri
- stop_instance = MiniTest::Mock.new.expect(:extract!, nil).expect(:save!, true)
- stop_class = MiniTest::Mock.new.expect :all, [stop_instance]
- @cli.__stop__ = stop_class
- @cli.run "extract sqlite::memory:"
- assert stop_instance.verify
- assert stop_class.verify
- end
-
def test_execute_help_command
@cli.run 'help'
assert_match /help/, @out.string
@@ -1,118 +0,0 @@
-require 'helper'
-
-class Model::Stop::ExtractorTest < MiniTest::Unit::TestCase
- def stub_extractor attributes
- Model::Stop::Extractor.new OpenStruct.new(attributes)
- end
-
- def test_extract!
- stop = OpenStruct.new :summary => 'Grand Ave East, Grand Av opposite approaching Ascot Av'
- extractor = Model::Stop::Extractor.new stop
- assert_same stop, extractor.extract!
- assert_equal 'Grand Av', stop.street1
- assert_equal 'Ascot Av', stop.street2
- assert_equal 'opposite_approaching', stop.locality
- end
-
- def test_street1_opposite_approaching
- assert_equal 'Grand Av', stub_extractor(:summary => 'Grand Ave East, Grand Av opposite approaching Ascot Av').street1
- end
-
- def test_street2_opposite_approaching
- assert_equal 'Ascot Av', stub_extractor(:summary => 'Grand Ave East, Grand Av opposite approaching Ascot Av').street2
- end
-
- def test_locality_opposite_approaching
- assert_equal 'opposite_approaching', stub_extractor(:summary => 'Grand Ave East, Grand Av opposite approaching Ascot Av').locality
- end
-
- def test_street1_opposite_far_side
- assert_equal 'Ipswich Rd', stub_extractor(:summary => 'P.A. Hospital - 14, Ipswich Rd opposite far side of Tottenham St').street1
- end
-
- def test_street2_opposite_far_side
- assert_equal 'Tottenham St', stub_extractor(:summary => 'P.A. Hospital - 14, Ipswich Rd opposite far side of Tottenham St').street2
- end
-
- def test_locality_opposite_far_side
- assert_equal 'opposite_far_side', stub_extractor(:summary => 'P.A. Hospital - 14, Ipswich Rd opposite far side of Tottenham St').locality
- end
-
- def test_street1_opposite
- assert_equal 'Partridge St', stub_extractor(:summary => 'Partridge, Partridge St opposite Tang St').street1
- end
-
- def test_street2_opposite
- assert_equal 'Tang St', stub_extractor(:summary => 'Partridge, Partridge St opposite Tang St').street2
- end
-
- def test_locality_opposite
- assert_equal 'opposite', stub_extractor(:summary => 'Partridge, Partridge St opposite Tang St').locality
- end
-
- def test_street1_appraoching
- assert_equal 'Blunder Rd', stub_extractor(:summary => 'Stop 66, Blunder Rd (approaching Ipswich Rd)').street1
- end
-
- def test_street2_appraoching
- assert_equal 'Ipswich Rd', stub_extractor(:summary => 'Stop 66, Blunder Rd (approaching Ipswich Rd)').street2
- end
-
- def test_locality_appraoching
- assert_equal 'approaching', stub_extractor(:summary => 'Stop 66, Blunder Rd (approaching Ipswich Rd)').locality
- end
-
- def test_street1_far_side
- assert_equal 'Illaweena St', stub_extractor(:summary => 'Waterstone, Illaweena St far side of Waterbrooke Crt').street1
- end
-
- def test_street2_far_side
- assert_equal 'Waterbrooke Crt', stub_extractor(:summary => 'Waterstone, Illaweena St far side of Waterbrooke Crt').street2
- end
-
- def test_locality_far_side
- assert_equal 'far_side', stub_extractor(:summary => 'Waterstone, Illaweena St far side of Waterbrooke Crt').locality
- end
-
- def test_street1_at
- assert_equal 'South East Busway', stub_extractor(:summary => 'Woolloongabba Platform 1, South East Busway At Main St').street1
- end
-
- def test_street2_at
- assert_equal 'Main St', stub_extractor(:summary => 'Woolloongabba Platform 1, South East Busway At Main St').street2
- end
-
- def test_locality_at
- assert_equal 'at', stub_extractor(:summary => 'Woolloongabba Platform 1, South East Busway At Main St').locality
- end
-
- def test_street1_near
- assert_equal 'Old Northern Rd', stub_extractor(:summary => 'Allamanda, Old Northern Rd (Near Allamanda Cres)').street1
- end
-
- def test_street2_near
- assert_equal 'Allamanda Cres', stub_extractor(:summary => 'Allamanda, Old Northern Rd (Near Allamanda Cres)').street2
- end
-
- def test_locality_near
- assert_equal 'near', stub_extractor(:summary => 'Allamanda, Old Northern Rd (Near Allamanda Cres)').locality
- end
-
- def test_street1_stips_stop_numbers
- assert_equal 'Woogaroo St', stub_extractor(:summary => '167 Woogaroo St (far side of Roxwell St)').street1
- assert_equal 'Serviceton St', stub_extractor(:summary => 'Stop 78, 79 Serviceton St (approaching Viola St)').street1
- assert_equal 'Glenala St', stub_extractor(:summary => 'Stop 71, 19-21 Glenala St (approaching Bampo St)').street1
- end
-
- def test_street1_without_locality
- assert_equal 'Oyster Cove - Hail And Ride', stub_extractor(:name => 'Oyster Cove - Hail And Ride').street1
- end
-
- def test_street2_without_locality
- assert_nil stub_extractor(:name => 'Oyster Cove - Hail And Ride').street2
- end
-
- def test_locality_without_locality
- assert_nil stub_extractor(:name => 'Oyster Cove - Hail And Ride').locality
- end
-end
@@ -20,18 +20,4 @@ def test_all
assert_equal ['Adelaide St', 'Queen St', 'Elizabeth St'], stops.map(&:name)
end
end
-
- def test_extract!
- stop = Model::Stop.new
- extractor_instance = MiniTest::Mock.new.expect :extract!, stop
- extractor_class = MiniTest::Mock.new.expect :new, extractor_instance, [stop]
- stop.__extractor__ = extractor_class
- stop.extract!
- assert extractor_instance.verify
- assert extractor_class.verify
- end
-
- def test__extractor__
- assert_equal Model::Stop::Extractor, Model::Stop.new.__extractor__
- end
end

0 comments on commit c7904f4

Please sign in to comment.