Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 100 lines (84 sloc) 2.739 kB
f96f03b @mislav add basic search functionality
authored
1 task :environment do
2 require_relative 'app'
3 end
4
7e932ce @mislav describe available rake tasks
authored
5 desc %(Create the database, prepare schema, import RFC index)
4cb8d0a @mislav script/bootstrap
authored
6 task :bootstrap => [:'db:bootstrap', :import_index, :import_popular]
69d837d @mislav rake bootstrap
authored
7
f96f03b @mislav add basic search functionality
authored
8 namespace :db do
7e932ce @mislav describe available rake tasks
authored
9 desc %(Rebuild the db schema)
f96f03b @mislav add basic search functionality
authored
10 task :rebuild => :environment do
11 DataMapper.auto_migrate!
12 end
13
7e932ce @mislav describe available rake tasks
authored
14 desc %(Automatically migrate the db schema based on changed model attributes)
f96f03b @mislav add basic search functionality
authored
15 task :migrate => :environment do
16 DataMapper.auto_upgrade!
17 end
4cb8d0a @mislav script/bootstrap
authored
18
7e932ce @mislav describe available rake tasks
authored
19 desc %(Create and upgrade the database schema if necessary)
4cb8d0a @mislav script/bootstrap
authored
20 task :bootstrap => :environment do
21 if RfcEntry.storage_exists?
22 Rake::Task[:'db:migrate'].invoke
23 else
24 Rake::Task[:'db:rebuild'].invoke
25 end
26 end
f96f03b @mislav add basic search functionality
authored
27 end
28
7e932ce @mislav describe available rake tasks
authored
29 desc %(Import the complete RFC index into the database)
f96f03b @mislav add basic search functionality
authored
30 task :import_index => ['tmp/rfc-index.xml', :environment] do |task|
31 require 'nokogiri'
32 require 'active_support/core_ext/object/try'
d94e174 @mislav improve RfcEntry storage model
authored
33 require 'date'
f96f03b @mislav add basic search functionality
authored
34
4cb8d0a @mislav script/bootstrap
authored
35 DataMapper.logger.set_log($stderr, :warn)
f96f03b @mislav add basic search functionality
authored
36
37 index = Nokogiri File.open(task.prerequisites.first)
d94e174 @mislav improve RfcEntry storage model
authored
38 num = 0
39
40 date_from_xml = ->(xml_date) {
41 if xml_date
42 year = xml_date.at('./year').text
43 month_name = xml_date.at('./month').text
44 day = xml_date.at('./day').try(:text)
45 Date.parse [year, month_name, day].join(' ')
46 end
47 }
f96f03b @mislav add basic search functionality
authored
48
49 index.search('rfc-entry').each do |xml_entry|
d94e174 @mislav improve RfcEntry storage model
authored
50 doc_id = xml_entry.at('./doc-id').text
51 unless entry = RfcEntry.get(doc_id)
52 entry = RfcEntry.new
53 entry.document_id = doc_id
54 entry.title = xml_entry.at('./title').text
55 entry.abstract = xml_entry.at('./abstract').try(:inner_html)
56 entry.keywords = xml_entry.search('./keywords/*').map(&:text)
57 end
58 entry.obsoleted = xml_entry.search('./obsoleted-by').any?
59 entry.publish_date = date_from_xml.(xml_entry.at('./date'))
60 num += 1 if entry.dirty?
9b476ad @mislav implement automatic fetching & rendering of RFCs
authored
61 entry.save
f96f03b @mislav add basic search functionality
authored
62 end
d94e174 @mislav improve RfcEntry storage model
authored
63
9b476ad @mislav implement automatic fetching & rendering of RFCs
authored
64 puts "updated #{num} entries (%d in database)." % RfcEntry.count
f96f03b @mislav add basic search functionality
authored
65 end
66
67 file 'tmp/rfc-index.xml' do |task|
68 mkdir_p 'tmp'
69 index_url = 'ftp://ftp.rfc-editor.org/in-notes/rfc-index.xml'
70 sh 'curl', '-#', index_url, '-o', task.name
71 end
b9dcf68 @mislav sort RFC search results by popularity score first
authored
72
7e932ce @mislav describe available rake tasks
authored
73 desc %(Update the RFCs in the database with a popularity score)
b9dcf68 @mislav sort RFC search results by popularity score first
authored
74 task :import_popular => :environment do
75 require 'nokogiri'
76 require 'open-uri'
77
78 popular = []
79 pop_url = 'http://www.faqs.org/rfc-pop%d.html'
9b476ad @mislav implement automatic fetching & rendering of RFCs
authored
80 num = 0
b9dcf68 @mislav sort RFC search results by popularity score first
authored
81
82 (1..5).each do |n|
83 html = Nokogiri open(pop_url % n)
84 html.search('#fmaincolumn a[href^="/rfcs/"]').each do |link|
85 popular << File.basename(link['href'], '.html')
86 end
87 end
88
89 popular.each_with_index do |name, idx|
9b476ad @mislav implement automatic fetching & rendering of RFCs
authored
90 if entry = RfcEntry.get(name)
b9dcf68 @mislav sort RFC search results by popularity score first
authored
91 entry.popularity = idx + 1
9b476ad @mislav implement automatic fetching & rendering of RFCs
authored
92 entry.save
93 num += 1
b9dcf68 @mislav sort RFC search results by popularity score first
authored
94 else
95 warn "could not find #{name}"
96 end
97 end
9b476ad @mislav implement automatic fetching & rendering of RFCs
authored
98 puts "applied popular score to #{num} entries."
b9dcf68 @mislav sort RFC search results by popularity score first
authored
99 end
Something went wrong with that request. Please try again.