Permalink
Browse files

Removed encoding detection - was causing troubles and we should keep …

…this on the user anyway
  • Loading branch information...
1 parent 05f1a82 commit 4e23129b372d2ed6546ae2e8dc80f47b92af3162 @Stiivi committed Apr 11, 2012
Showing with 6 additions and 15 deletions.
  1. +6 −15 brewery/ds/csv_streams.py
View
@@ -126,8 +126,8 @@ class CSVDataSource(base.DataSource):
Some code taken from OKFN Swiss library.
"""
def __init__(self, resource, read_header=True, dialect=None, encoding=None,
- detect_encoding=False, detect_header=False, sample_size=200,
- skip_rows=None, empty_as_null=True,fields=None, **reader_args):
+ detect_header=False, sample_size=200, skip_rows=None,
+ empty_as_null=True,fields=None, **reader_args):
"""Creates a CSV data source stream.
:Attributes:
@@ -136,8 +136,6 @@ def __init__(self, resource, read_header=True, dialect=None, encoding=None,
or not. ``True`` by default.
* encoding: source character encoding, by default no conversion is
performed.
- * detect_encoding: read sample from source and determine whether
- source is UTF8 or not
* detect_headers: try to determine whether data source has headers
in first row or not
* sample_size: maximum bytes to be read when detecting encoding
@@ -152,12 +150,9 @@ def __init__(self, resource, read_header=True, dialect=None, encoding=None,
"""
self.read_header = read_header
self.encoding = encoding
- self.detect_encoding = detect_encoding
self.detect_header = detect_header
self.empty_as_null = empty_as_null
- self._autodetection = detect_encoding or detect_header
-
self.sample_size = sample_size
self.resource = resource
self.reader_args = reader_args
@@ -191,18 +186,14 @@ def initialize(self):
handle = None
- if self._autodetection:
+ if self.detect_header:
sample = self.file.read(self.sample_size)
# Encoding test
- if self.detect_encoding and type(sample) == unicode:
- self.encoding = "utf-8"
-
- if self.detect_header:
- sample = sample.encode('utf-8')
- sniffer = csv.Sniffer()
- self.read_header = sniffer.has_header(sample)
+ sample = sample.encode('utf-8')
+ sniffer = csv.Sniffer()
+ self.read_header = sniffer.has_header(sample)
self.file.seek(0)

0 comments on commit 4e23129

Please sign in to comment.