@@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = %q{Html2Feedbooks} - s.version = '1.0.5' + s.version = '1.0.6' s.date = %q{2009-04-28} s.authors = ["Benoit Larroque"] s.email = "zeta dot ben at gmail dot com" Html2Feedbooks.gemspec @@ -14,6 +14,7 @@ include HTML2FB options = {} options[:conf] = "conf.yaml" options[:preview] = true +options[:conv] = true OptionParser.new do |opts| opts.banner = "Usage: html2fb [options] URL" @@ -23,6 +24,9 @@ OptionParser.new do |opts| opts.on("-s", "-s","Send to feedbooks") do |f| options[:preview] = !f end + opts.on("-nc", "--no-conv","No charset conversion") do |f| + options[:conv] = !f + end end.parse! valid=false @@ -40,7 +44,7 @@ while !valid print "URL : " if entry.nil? || entry=='' entry=STDIN.readline.strip unless valid end -conf=Conf.new(options[:conf]) +conf=Conf.new(options[:conf],options[:conv]) content=Downloader.download(url) #puts content.size doc=Parser.new(conf).parse(content) bin/html2fb.rb @@ -2,13 +2,14 @@ require 'yaml' module HTML2FB class Conf - def initialize(file) + def initialize(file,conv) ['','./',"#{File.dirname(__FILE__)}/","#{File.dirname(__FILE__)}/../confs/"].each do |p| f=p+file begin if File.readable?(f) && File.exists?(f) @conf=File.open(f,'r'){|txt| YAML::load(txt)} puts "loaded config file : "+f + @conf['conv']=conv return end rescue Exception => e lib/conf.rb @@ -15,6 +15,25 @@ module HTML2FB def parse(txt) puts "Parsing HTML" pdoc=Hpricot(txt) + if @conf['conv'] + mc=pdoc/'meta[@http-equiv="Content-Type"]' + if mc.size>0 + charset=mc.first.attributes['content'].split(';').find do |s| + s.strip[0,7]=='charset' + end + unless charset.nil? + tc=charset.split('=').last.strip + end + + unless tc.nil? + puts "Trying to convert source encoding from #{tc} to utf-8" + require 'iconv' + pdoc=Hpricot(Iconv.conv('utf-8',tc.downcase,txt)) + + end + + end + end doc=Document.new puts "Removing garbage elements" remove_objs(pdoc) lib/parser.rb 19636e81d03b2a0065f3afa29784714d86a3a2cb Benoit Larroque zeta.ben@gmail.com http://github.com/zetaben/Html2Feedbooks/commit/181fed872dd4f203b80204cd33035d2fbe31132e 181fed872dd4f203b80204cd33035d2fbe31132e 2009-09-23T12:58:09-07:00 2009-09-23T12:58:09-07:00 Adding an heuristic to autoconvert non UTF-8 files 326515934c2697dc3be28b025b27b1ec4ea53dc0 Benoit Larroque zeta.ben@gmail.com