Mapping checker added

Gemfile

@@ -27,6 +27,9 @@ gem 'turbolinks'
 # Used for bible scraping
 gem 'nokogiri'
 
+# Colorizing console output
+gem 'colored'
+
 group :development, :test do 
   gem 'pry'
 end

Gemfile.lock

@@ -48,6 +48,7 @@ GEM
       coffee-script-source
       execjs
     coffee-script-source (1.7.0)
+    colored (1.2)
     connection_pool (2.0.0)
     coveralls (0.7.0)
       multi_json (~> 1.3)
@@ -171,6 +172,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  colored
   coveralls
   factory_girl_rails
   jquery-rails

lib/bible/scrapers/base.rb

@@ -12,6 +12,13 @@ def scrape
           Bible::Book.all.map { |book| scrape_book(book) }
         end
 
+        # Returns the list of books titles that require mapping
+        #
+        # @return [ Array ] array of book titles needed to map
+        def missing_mappings
+          books_with_missing_mapping.map(&:title)
+        end
+
         private
 
         # Srapes single book translation
@@ -54,7 +61,7 @@ def process_verse(book, chapter, verse_number)
         # 
         # @return [ String ] mapping value
         def book_mapping(book)
-          title = book.title.downcase
+          title = book.title.downcase.gsub(/\s+/, '')
           CONFIG[:sources][translation][:mappings][title] || title
         end
 
@@ -97,6 +104,31 @@ def url
         def translation
           @translation ||= name.underscore.split('/').last
         end
+
+        # Returns the list of books titles that require mapping
+        #
+        # @return [ Array ] array of books needed to map
+        def books_with_missing_mapping
+          Bible::Book.all.select do |book|
+            mapping = book_mapping(book)
+            Rails.logger.info("Mapping Check: #{book.title}")
+            begin
+              out scrape_verse(mapping, 1, 1), true
+            rescue OpenURI::HTTPError
+              out false, true
+            end
+          end
+        end
+
+        # Outputs iteration result in a Rspec like format
+        #
+        # @param [ true, false ] revert - indicates whether it's needed to revert the result
+        #
+        # @retun [ true, false] result itself
+        def out(result, revert = false)
+          (result ? print('.'.green) : print('F'.red)) unless Rails.env.test?
+          revert ? !result : result
+        end
       end
     end
   end

lib/tasks/scraper.rake

@@ -0,0 +1,13 @@
+namespace :scraper do
+
+  desc 'Get the list of books that require mappings'
+  task :missing_mappings, [:translation] => [:environment] do |t, args|
+    if args[:translation].present?
+      puts 'Checking mappings. This can take a while...'
+      result = Bible::Scrapers.const_get(args[:translation].camelize).missing_mappings.to_sentence
+      puts "\nRequire mapping: #{result}"
+    else
+      puts 'Please set the translation like: rake scraper:missing_mappings[ru]'
+    end
+  end
+end

spec/libs/bible/scrapers/base_spec.rb

@@ -149,6 +149,18 @@
 
   describe '#missing_mappings' do
 
-    it 'returns the list of books that need mapping'
+    let(:genesis) { build(:book, title: 'Gn') } 
+    let(:leviticus) { build(:book, title: 'Le') } 
+
+    before do
+      subject.stub(translation: 'ru')
+      subject.stub(:scrape_verse).with('ge', 1, 1).and_return('Genesis')
+      subject.stub(:scrape_verse).with('le', 1, 1).and_return(nil) 
+      Bible::Book.stub(all: [ genesis, leviticus ])   
+    end
+
+    it 'returns the list of books that need mapping' do
+      expect(subject.missing_mappings).to eq([ 'Le' ])
+    end
   end
 end

spec/libs/bible/scrapers/ru_spec.rb

@@ -20,15 +20,15 @@
       context 'when the verse number exists' do
 
         it 'returns verse text' do
-          pending 'API checks are suspended on CI' if ENV['TRAVIS']
+          pending 'integrated API checks are suspended due to performance reasons'
           expect(subject.send(:scrape_verse, 'ge', 1, 1)).to eq("В начале сотворил Бог небо и землю.")
         end
       end
 
       context 'when the verse number doesnt exist' do
 
         it 'returns nil' do
-          pending 'API checks are suspended on CI' if ENV['TRAVIS']
+          pending 'integrated API checks are suspended due to performance reasons'
           expect(subject.send(:scrape_verse, 'ge', 1, 100)).to be_nil
         end
       end