Added suica data scripts (previously in separate repository)

data/suica/Gemfile

@@ -0,0 +1,7 @@
+source :rubygems
+
+gem 'sqlite3'
+gem 'redis'
+gem 'nestful'
+gem 'graticule'
+gem 'unicode_utils'

data/suica/Gemfile.lock

@@ -0,0 +1,30 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (3.2.0.rc1)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    graticule (2.2.0)
+      activesupport
+      happymapper (>= 0.3.0)
+      i18n
+      json
+    happymapper (0.4.0)
+      libxml-ruby (~> 2.0)
+    i18n (0.6.0)
+    json (1.6.4)
+    libxml-ruby (2.2.2)
+    multi_json (1.0.4)
+    nestful (0.0.7)
+      activesupport (>= 3.0.0.beta)
+    redis (2.2.2)
+    sqlite3 (1.3.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  graticule
+  nestful
+  redis
+  sqlite3

data/suica/create-sqlite-db.rb

@@ -0,0 +1,173 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'rubygems'
+require 'csv'
+require 'sqlite3'
+require 'redis'
+require "unicode_utils/nfkc"
+require './lib/gtrans'
+require './lib/gcode'
+
+COLS = %w(AreaCode LineCode StationCode CompanyName CompanyName_en LineName LineName_en StationName StationName_en StationNote Latitude Longitude)
+BUS_COLS = %w(LineCode StationCode CompanyName LineName StationName Note CompanyName_en LineName_en StationName_en)
+
+DB_OUT  = '../out/StationCode.db'
+CSV_OUT = '../out/StationCode.csv'
+
+RAIL_TABLE = 'StationCode'
+BUS_TABLE  = 'IruCaStationCode'
+
+VERSION = 3
+
+@redis = Redis.new
+
+@english = {
+  # Lines
+  '大阪環状'           => 'Ōsaka Loop',
+  '白新線'             => 'Hakushin',
+  '鴨東'               => 'Ōtō',
+  '東西'               => 'Tōzai',
+  '総武本'             => 'Sōbu',
+  '中央本'             => 'Chūō',
+  '東京モノレール羽田' => 'Tokyo Monorail',
+  '東京臨海新交通臨海' => 'Tokyo Waterfront New Transit'
+}
+
+# Companies
+CSV.foreach('data/RailCompanies.csv', headers: :first_row) do |row|
+  ja_name = row['Japanese']
+  en_name = row['English (Fixed)']
+  @english[ja_name] = en_name
+end
+
+def translate(ja_text, translate = true, transliterate = false)
+  return @english[ja_text] if @english.include?(ja_text) 
+
+  translated, translit = google_translate(ja_text)
+
+  # Clean up transliteration
+  translit = translit.gsub(' ', '').gsub("'", '')
+
+  if translated.contains_cjk? or translated == ja_text or translit == translated
+    # Bad translation
+    translit
+  elsif translit.contains_cjk?
+    # Bad transliteration
+    translated
+  elsif ja_text.contains_kana?
+    # Always return translation if there's kana to avoid engrlish
+    translated
+  elsif transliterate and translate
+    "#{translit} (#{translated})"
+  elsif transliterate
+    translit
+  else
+    translated
+  end
+end
+
+def translate_company(ja_text)
+  en = translate(ja_text)
+  bad_suffixes = [ 'Corp.', 'Co., Ltd.', 'Corporation' ]
+  bad_suffixes.inject(en) { |en, bad| en.gsub(/ #{Regexp.escape(bad)}$/, '') }
+end
+
+def translate_line(ja_text)
+  line_num = UnicodeUtils.nfkc(ja_text).match(/(\d+)号/) {|m| m[1] }
+  if line_num
+    ja_text = ja_text.sub(/\d+号/, '')
+    ja_text = ja_text.sub('線', '')
+    return "##{line_num}" if ja_text.blank?
+  end
+  en = translate(ja_text, false, true)
+  line_num ? "##{line_num} #{en}" : en
+end
+
+def geocode_station(station_name)
+  geocode([ station_name, '駅' ].join(' '))
+end
+
+FileUtils.rm(CSV_OUT) if File.exists?(CSV_OUT)
+FileUtils.rm(DB_OUT) if File.exists?(DB_OUT)
+
+@db = SQLite3::Database.new(DB_OUT)
+@db.execute("PRAGMA user_version = #{VERSION}")
+@db.execute("CREATE TABLE #{RAIL_TABLE} (_id INTEGER PRIMARY KEY AUTOINCREMENT, #{COLS.join(',')})")
+@db.execute("CREATE INDEX line_sta_index ON #{RAIL_TABLE} (LineCode, StationCode)")
+@db.execute("CREATE INDEX area_line_sta_index ON #{RAIL_TABLE} (AreaCode, LineCode, StationCode)")
+@db.execute("CREATE TABLE 'android_metadata' ('locale' TEXT DEFAULT 'en_US')")
+@db.execute("INSERT INTO 'android_metadata' VALUES ('en_US')")
+
+CSV.open(CSV_OUT, "wb") do |csv|
+  csv << COLS
+  CSV.foreach('data/SFCardFan-RailStationCode.csv', headers: :first_row) do |row|
+    area_code    = row['AreaCode']
+    line_code    = row['LineCode']
+    station_code = row['StationCode']
+    company_name = row['CompanyName']
+    line_name    = row['LineName']
+    station_name = row['StationName']
+    station_note = row['Note']
+
+    company_name_en = translate_company(company_name)
+    line_name_en    = translate_line(line_name)
+    station_name_en = translate(station_name, false, true)
+
+    latitude, longitude = geocode_station(station_name)
+
+    new_row = [
+      area_code, 
+      line_code, 
+      station_code,
+      company_name, 
+      company_name_en,
+      line_name, 
+      line_name_en,
+      station_name, 
+      station_name_en,
+      station_note,
+      latitude,
+      longitude
+    ]
+
+    puts new_row.join(',')
+    csv << new_row
+
+    @db.execute("INSERT INTO #{RAIL_TABLE} (#{COLS.join(',')}) VALUES(#{COLS.length.times.map{'?'}.join(',')})", *new_row)
+  end
+end
+
+# NOTE: This data is from the nfc-felica project... the english has not been 
+# vetted at *all* and is probably a huge disaster.
+@db.execute("CREATE TABLE #{BUS_TABLE} (_id INTEGER PRIMARY KEY AUTOINCREMENT, #{BUS_COLS.join(',')})")
+@db.execute("CREATE INDEX iruca_line_sta_index ON #{BUS_TABLE} (LineCode, StationCode)")
+CSV.foreach('data/nfcfelica-iruca.csv', headers: :first_row) do |row|
+  line_code       = row['LineCode']
+  station_code    = row['StationCode']
+  company_name    = row['CompanyName']
+  line_name       = row['LineName']
+  station_name    = row['StationName']
+  note            = row['Note']
+  company_name_en = row['CompanyName_en']
+  line_name_en    = row['LineName_en']
+  station_name_en = row['StationName_en']
+
+  new_row = [
+    line_code,
+    station_code,
+    company_name,
+    line_name,
+    station_name,
+    note,
+    company_name_en,
+    line_name_en,
+    station_name_en
+  ]
+
+  puts new_row.join(',')
+
+  @db.execute("INSERT INTO #{BUS_TABLE} (#{BUS_COLS.join(',')}) VALUES(#{BUS_COLS.length.times.map{'?'}.join(',')})", *new_row)
+end
+
+puts "Wrote #{DB_OUT} and #{CSV_OUT}"

data/suica/data/RailCompanies.csv

@@ -0,0 +1,60 @@
+Japanese,English (Google Translate),English (Fixed)
+愛知環状鉄道,Aichi Loop Railway,Aichi Loop Line Co.
+青い森鉄道,Blue Forest Railway,Aoimori Railway
+秩父鉄道,Chichibu,Chichibu Railway
+叡山電鉄,Railway Eizan,Eizan Electric Railway
+北海道ちほく高原鉄道,"Sure plateau railroad almost immediately , Hokkaido",Hokkaido Chihoku Kougen Railway
+北神急行電鉄,Northern Railway express God,Hokushin Kyuko Railway
+北総鉄道,Total Northern Railway,Hokuso Railway
+茨城交通,Ibaraki Transportation,Ibaraki Kotsu Co.
+IGRいわて銀河鉄道,IGR Iwate Galaxy,IGR Iwate Galaxy Railway
+上信電鉄,Railway communication on,Joshin Electric Railway
+四国旅客鉄道,Shikokuryokakutetsudō,JR Shikoku
+蒲原鉄道,Kanbaratetsudō,Kanbara Tetsudo Co.
+京浜急行電鉄,Keihin Electric Express Railway,Keikyu Corp.
+京王電鉄,Keio,Keio Corp.
+近畿日本鉄道,Kintetsu,Kintetsu Corp.
+北大阪急行電鉄,Northern Osaka Express Railway,Kita-Osaka Kyuko Railway
+北近畿タンゴ鉄道,Kita Kinki Tango Railway,Kitakinki Tango Railway
+神戸高速鉄道,Kobe high-speed rail,Kobe Rapid Transit Railway
+舞浜リゾートライン,Resort Line Maihama,Maihama Resort Line
+首都圏新都市鉄道,Shutokenshintoshitetsudō,Metropolitan Intercity Railway
+名古屋臨海高速鉄道,Nagoya critical high-speed rail,Nagoya Seaside Rapid Railway
+錦川鉄道,Nishikikawatetsudō,Nishikigawa Railway
+能勢電鉄,Nose Electric Railway,Nosé Electric Railway
+のと鉄道,And the railroad,Noto Railway
+岡山電気軌道,Electrical Okayama track,Okayama Electric Tramway
+大阪高速鉄道,Osaka High Speed ​​Rail,Osaka Monorail Co.
+大阪府都市開発,Osaka Urban Development,Osaka Prefectural Urban Development Co.
+総武流山電鉄,Nagareyama Sobu Railway,Ryutetsu
+埼玉高速鉄道,Saitama High Speed ​​Railway,Saitama Railway
+三岐鉄道,Three branch railway,Sangi Railway
+仙台空港鉄道,Sendai Airport Train,Sendai Airport Transit
+静岡鉄道,Shizuokatetsudō,Shizuoka Railway
+多摩都市モノレール,Tama Monorail,Tama Toshi Monorail
+桃花台新交通,New transportation peach garden tub,Tokadai New Transit
+東海交通事業,Transportation projects Tokai,Tokai Transport Service Co.
+東京地下鉄,Tokyo subway,Tokyo Metro
+東葉高速鉄道,High-speed train leaves East,Toyo Rapid Railway
+豊橋鉄道,Toyohashitetsudō,Toyohashi Railway
+和歌山電鐵,Wakayama Electric Iron,Wakayama Electric Railway
+わたらせ渓谷鐵道,We span the valley railway,Watarase Keikoku Railway
+野岩鉄道,Train rocks,Yagan Railway
+山万,Mountain ten thousand,Yamaman
+横浜高速鉄道,YOKOHAMA high-speed rail,Yokohama Minatomirai Railway
+横浜新都市交通,New Yokohama city traffic,Yokohama New Transit
+九州旅客鉄道,Kyusyu Railway Co.,JR Kyusyu
+東海旅客鉄道,,JR Central
+北海道旅客鉄道,,JR Hokkaido
+西日本旅客鉄道,West Japan Railway,JR West
+JR東日本,JR東日本,JR East
+東日本旅客鉄道,East Japan Railway,JR East
+東京都交通局,Tokyo Metropolitan Bureau of Transportation,Toei
+福岡市交通局,Fukuoka City Transportation Bureau,Fukuoka Subway
+神戸市交通局,Kobe Municipal Transportation Bureau,Kobe Subway
+京都市交通局,Kyoto Municipal Transportation Bureau,Kyoto Subway
+大阪市交通局,Osaka Municipal Transportation Bureau,Osaka Subway
+札幌市交通局,Sapporo City Transportation Bureau,Sapporo Subway
+仙台市交通局,Sendai City Transportation Bureau,Sendai Subway
+名古屋市交通局,Transportation Bureau City of Nagoya,Nagoya Subway
+横浜市交通局,Yokohama City Transportation Bureau,Yokohama Subway