Update scraper.py

scraper.py

@@ -5,6 +5,57 @@
 import scraperwiki
 from tqdm import tqdm
 from datetime import datetime
+from __future__ import with_statement
+import sys
+import codecs
+from chardet.universaldetector import UniversalDetector
+
+
+def get_encoding_type(current_file):
+    detector.reset()
+    for line in file(current_file):
+        detector.feed(line)
+        if detector.done: break
+    detector.close()
+    return detector.result['encoding']
+
+
+def convertFileBestGuess(filename):
+    targetFormat = 'utf-8'
+    outputDir = os.path.join('downloads')
+    detector = UniversalDetector()
+    sourceFormats = ['ascii', 'iso-8859-1']
+    for format in sourceFormats:
+        try:
+            with codecs.open(fileName, 'rU', format) as sourceFile:
+                writeConversion(sourceFile)
+                print('Done.')
+                return
+        except UnicodeDecodeError:
+            pass
+
+
+def convertFileWithDetection(fileName):
+    targetFormat = 'utf-8'
+    outputDir = os.path.join('downloads')
+    detector = UniversalDetector()
+    print("Converting '" + fileName + "'...")
+    format = get_encoding_type(fileName)
+    try:
+        with codecs.open(fileName, 'rU', format) as sourceFile:
+            writeConversion(sourceFile)
+            print('Done.')
+            return
+    except UnicodeDecodeError:
+        pass
+
+    print("Error: failed to convert '" + fileName + "'.")
+
+
+def writeConversion(file):
+    with codecs.open(outputDir + '/' + fileName, 'w', targetFormat) as targetFile:
+        for line in file:
+            targetFile.write(line)
 
 
 def download_file(url, file_name):
@@ -35,6 +86,9 @@ def process_file_debentures(url):
     download_file(url, path_file)
     # process file
     print('Processando arquivo', name_file)
+    # convert file to UTF-8    
+    convertFileWithDetection(path_file)
+    # process file 
     process_file(path_file)
     # remove processed file
     os.remove(path_file)
@@ -48,13 +102,6 @@ def process_file(file_path):
         sep='\t'
     )
 
-    df.to_csv(file_path, encoding='utf-8-sig')
-
-    df = pd.read_csv(
-        file_path,
-        encoding='utf-8-sig'
-    )
-
     print('Importing {} items'.format(len(df)))
 
     # converte para datetime