fixes for bad quotes in csv

albertobraschi · Nov 21, 2008 · ab565db · ab565db
1 parent f539519
commit ab565db
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 7 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,5 @@
+v0.1.5. fixes for bad quotes in csv causing fastercsv to fail
+
 v0.1.4. now use fastercsv to parse table row data
 
 v0.1.3. fixed bug due to change in google's graph csv date labels

diff --git a/lib/rugalytics.rb b/lib/rugalytics.rb
@@ -12,7 +12,7 @@
 
 # See README for usage documentation.
 module Rugalytics
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 
   FORMAT_PDF = '0' unless defined? FORMAT_PDF
   FORMAT_XML = '1' unless defined? FORMAT_XML

diff --git a/lib/rugalytics/report.rb b/lib/rugalytics/report.rb
@@ -95,7 +95,24 @@ def handle_tables lines
         morph(items_attribute, items)
 
         while (values_line = lines[index]) && values_line[/^# -/].nil? && values_line.strip.size > 0
-          values = FasterCSV.parse_line(values_line)
+          begin
+            values = FasterCSV.parse_line(values_line)
+          rescue Exception => e
+            # fix broken csv
+            values_line.gsub!(/,""/,',"\"')
+            values_line.gsub!(/"",/,'\"",')
+            values_line.gsub!(/^""/,'"\"')
+            values_line.gsub!(/""$/,'\""')
+            values_line.gsub!(/([^\\])""/,'\1\"')
+            values_line.gsub!(/\\"/,'""')
+            begin
+              values = FasterCSV.parse_line(values_line)
+            rescue Exception => e
+              values_line.gsub!(/\\"/,'')
+              values_line.gsub!(/"/,'')
+              values = FasterCSV.parse_line(values_line)
+            end
+          end
           items << Item.new(attributes, values, base_url)
           index = index.next
         end

diff --git a/lib/rugalytics/server.rb b/lib/rugalytics/server.rb
@@ -20,16 +20,19 @@ def initialize
       end
 
       @profile = Rugalytics.default_profile
+      @reports = {}
 
       server = HTTPServer.new :Port => 8888
 
       server.mount("/", Rugalytics::Servlet)
 
       server.mount_proc("/top_content_detail_keywords") {|request, response|
         url = request.query['url']
-        report = @profile.top_content_detail_keywords_report(:url => url)
-        response.body = [url, report.name, report.items.to_yaml].join("\n")
-        response['Content-Type'] = "text/plain"
+        @reports[url] ||= @profile.top_content_detail_keywords_report(:url => url)
+        items = @reports[url].items
+        data = {:url =>url, :report_name=>@reports[url].name, :items=>items}
+        response.body = data.to_json
+        response['Content-Type'] = "application/json"
       }
 
       trap("INT"){ server.shutdown }

diff --git a/spec/lib/rugalytics/report_spec.rb b/spec/lib/rugalytics/report_spec.rb
@@ -100,7 +100,7 @@
         @base_url = %Q|your_site.com|
         @attributes = %Q|URL,Page Views,Unique Page Views,Time on Page,Bounce Rate,% Exit,$ Index, Keyword|
         @values1 = %Q|/,189,157,54.94957983193277,0.4862385392189026,0.37037035822868347,0.0,"ABC, Project"|
-        @values2 = %Q|/bills,60,38,54.17307692307692,0.0,0.13333334028720856,0.0,shoes|
+        @values2 = %Q|/bills,60,38,54.17307692307692,0.0,0.13333334028720856,0.0,""new zealand"" house ""begins with h""|
         @csv = ['# ----------------------------------------',
                 @base_url,
                 'Top Content,',
@@ -119,7 +119,7 @@
         item1 = mock('item1')
         item2 = mock('item2')
         array1 = ['/','189','157','54.94957983193277','0.4862385392189026','0.37037035822868347','0.0',"ABC, Project"]
-        array2 = ['/bills','60','38','54.17307692307692','0.0','0.13333334028720856','0.0','shoes']
+        array2 = ['/bills','60','38','54.17307692307692','0.0','0.13333334028720856','0.0','"new zealand" house "begins with h"']
         Item.should_receive(:new).with(@attributes.split(','), array1, @base_url).and_return item1
         Item.should_receive(:new).with(@attributes.split(','), array2, @base_url).and_return item2