Skip to content

Commit

Permalink
Added sbility to start scraping from particular book
Browse files Browse the repository at this point in the history
  • Loading branch information
k-rudy committed Apr 23, 2014
1 parent 71d2a3f commit ca0829d
Show file tree
Hide file tree
Showing 6 changed files with 43 additions and 9 deletions.
2 changes: 2 additions & 0 deletions app/models/bible/book.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ class Bible::Book
index order: 1

default_scope ->{ asc(:order) }
scope :by_title, ->(title) { where(title: title) }
scope :starting_from, ->(book_order) { where(order: { '$gte' => book_order }) }

# Order of the latest book verse
#
Expand Down
2 changes: 1 addition & 1 deletion app/models/bible/verse.rb
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
class Bible::Verse
include Mongoid::Document

belongs_to :book
belongs_to :book, class_name: 'Bible::Book'

field :chapter, type: Integer
field :text, localize: true
Expand Down
4 changes: 2 additions & 2 deletions config/books.yml
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,7 @@ bible:
full: Sirah
ru:
short: Сир
full: Книга премудрости Иисуса, сына Сирахова
full: 'Книга премудрости Иисуса, сына Сирахова'
pl:
short:
full:
Expand Down Expand Up @@ -510,7 +510,7 @@ bible:
by:
short: Бар
full: Кніга прарока Баруха
chapters: 6
chapters: 7
ezekiel:
order: 33
title:
Expand Down
16 changes: 13 additions & 3 deletions lib/bible/scrapers/base.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ class Base
class <<self
# Scrapes all Bible translation to the db
#
def scrape
Bible::Book.all.map { |book| scrape_book(book) }
# @param [ String ] book title to start scraping from
def scrape(book_to_start_from = nil)
books_to_scrape(book_to_start_from).map { |book| scrape_book(book) }
end

# Returns the list of books titles that require mapping
Expand All @@ -21,14 +22,23 @@ def missing_mappings

private

def books_to_scrape(book_to_start_from)
if book_to_start_from
book = Bible::Book.by_title(book_to_start_from).first
Bible::Book.starting_from(book.order)
else
Bible::Book.all
end
end

# Srapes single book translation
#
# @param [ Bible::Book ] book
def scrape_book(book)
print "\nScraping #{book.title}:" unless Rails.env.test?
book.chapters_count.times do |i|
print "\n#{i + 1}: " unless Rails.env.test?
scrape_chapter(book, i + 1)
scrape_chapter(book, i + 1)
end
end

Expand Down
4 changes: 2 additions & 2 deletions lib/tasks/scraper.rake
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ namespace :scraper do
end

desc 'Scrapes the given granslation of the Bible'
task :scrape, [:translation] => [:environment] do |t, args|
task :scrape, [:translation, :start_from] => [:environment] do |t, args|
if args[:translation].present?
result = Bible::Scrapers.const_get(args[:translation].camelize).scrape
result = Bible::Scrapers.const_get(args[:translation].camelize).scrape(args[:start_from])
else
puts 'Please set the translation like: rake scraper:scrape[ru]'
end
Expand Down
24 changes: 23 additions & 1 deletion spec/libs/bible/scrapers/base_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,30 @@
end

it 'returns the list of books that need mapping' do
#puts CONFIG.inspect
puts CONFIG.inspect
expect(subject.missing_mappings).to eq([ 'Le', 'Nm' ])
end
end

describe '#books_to_scrape' do

context 'when the book_to_start_from is present' do

let(:book) { double('book', order: 10, title: 'Ps') }
before { Bible::Book.stub(by_title: [ book ]) }

it 'returns books starting from that book' do
expect(Bible::Book).to receive(:starting_from).with(10)
subject.send(:books_to_scrape, 'Ps')
end
end

context 'when the book_to_start_from is nil' do

it 'returns all books' do
expect(Bible::Book).to receive(:all)
subject.send(:books_to_scrape, nil)
end
end
end
end

0 comments on commit ca0829d

Please sign in to comment.