Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

thread pool is unreliable; go single-threaded

  • Loading branch information...
commit b90315d2486fb21e9cbedd378ad1c5161da091b1 1 parent 2729afc
@danchoi authored
View
31 lib/vnews/config.rb
@@ -1,6 +1,5 @@
require 'vnews/sql'
require 'yaml'
-require 'thread_pool'
require 'vnews/constants'
require 'timeout'
@@ -87,29 +86,19 @@ def self.update_folders
puts "Adding feeds: #{(new_feeds - old_feeds).inspect}"
puts "Adding folder-feed associations: #{(ff - old_ff).inspect}"
- feeds2 = []
- pool = ThreadPool.new Vnews::POOLSIZE
- puts "Using thread pool size of 10"
- # TODO sometimes feeds are downloaded twice;
ff.each do |feed_url, folder|
- pool.process do
- begin
- Timeout::timeout(Vnews::TIMEOUT) do
- feeds2 << Vnews::Feed.fetch_feed(feed_url, folder)
+ begin
+ Timeout::timeout(Vnews::TIMEOUT) do
+ feed_url, f, folder = *Vnews::Feed.fetch_feed(feed_url, folder)
+ folder ||= "Misc"
+ if f.nil?
+ $stderr.print "\nNo feed found for #{feed_url}\n"
+ else
+ Vnews::Feed.save_feed(feed_url, f, folder)
end
- rescue Timeout::Error
- puts "TIMEOUT ERROR: #{feed_url}"
end
- end
- end
- pool.join
- feeds2.each do |x|
- feed_url, f, folder = *x
- folder ||= "Misc"
- if f.nil?
- $stderr.print "\nNo feed found for #{feed_url}\n"
- else
- Vnews::Feed.save_feed(feed_url, f, folder)
+ rescue Timeout::Error
+ puts "TIMEOUT ERROR: #{feed_url}"
end
end
$stderr.puts "\nDone."
View
1  lib/vnews/feed.rb
@@ -48,6 +48,7 @@ def self.save_feed(feed_url, f, folder=nil)
Vnews.sql_client.insert_item item.merge(:feed => feed_url, :feed_title => f[:meta][:title])
$stderr.print "."
end
+ print "\n"
rescue
puts "ERROR: dump: #{feed_url.inspect}, #{folder.inspect}: #{f.inspect} "
raise
View
22 lib/vnews/folder.rb
@@ -1,5 +1,4 @@
require 'vnews/feed'
-require 'thread_pool'
require 'vnews/constants'
class Vnews
@@ -12,25 +11,16 @@ def self.update_folder(folder)
require 'vnews/display'
folder = Vnews::Display.strip_item_count(folder)
puts "Updating folder: #{folder.inspect}"
- feeds = []
- pool = ThreadPool.new Vnews::POOLSIZE
- puts "Using thread pool size of #{Vnews::POOLSIZE}"
Vnews.sql_client.feeds_in_folder(folder.strip).each do |feed|
- pool.process do
- begin
- Timeout::timeout(Vnews::TIMEOUT) do
- feeds << Vnews::Feed.fetch_feed(feed, folder)
- end
- rescue Timeout::Error
- puts "TIMEOUT ERROR: #{feed_url}"
+ begin
+ Timeout::timeout(Vnews::TIMEOUT) do
+ feed_url, f, folder = *Vnews::Feed.fetch_feed(feed, folder)
+ Vnews::Feed.save_feed feed_url, f, folder
end
+ rescue Timeout::Error
+ puts "TIMEOUT ERROR: #{feed_url}"
end
end
- pool.join
- puts "Saving data to database"
- feeds.select {|x| x[1]}.compact.each do |feed_url, f, folder|
- Vnews::Feed.save_feed feed_url, f, folder
- end
puts "\nDone"
end
end
View
2  lib/vnews/version.rb
@@ -1,4 +1,4 @@
class Vnews
- VERSION = '0.4.1'
+ VERSION = '0.4.2'
end
Please sign in to comment.
Something went wrong with that request. Please try again.