fix bug 1037537 - encode utf-8 properly, and remove NULL characters w…

…hich postgres will not allow to be imported into TEXT NOT NULL columns
mozilla-services · Jul 11, 2014 · 372cfb1 · 372cfb1
1 parent e32f3dd
commit 372cfb1
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/socorro/cron/jobs/fetch_adi_from_hive.py b/socorro/cron/jobs/fetch_adi_from_hive.py
@@ -169,7 +169,10 @@ def run(self, connection, date):
                         f.write(
                             "\t"
                             .join(
-                                urllib2.unquote(str(v)).replace('\\', '\\\\')
+                                urllib2.unquote(v.encode('utf-8', 'ignore'))
+                                    .replace('\\', '\\\\')
+                                    .replace('\0', '')
+                                if isinstance(v, basestring) else str(v)
                                 for v in row
                             )
                         )

diff --git a/socorro/unittest/cron/jobs/test_fetch_adi_from_hive.py b/socorro/unittest/cron/jobs/test_fetch_adi_from_hive.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
@@ -73,7 +74,7 @@ def return_test_data(fake):
                  1],
                 ['2019-01-01',
                  'NothingMuch',
-                 'Ginko',
+                 u'Ginko☢\0',
                  '2.3.2',
                  '10.0.5a',
                  'release',
@@ -149,7 +150,7 @@ def return_test_data(fake):
             }, {
                 'report_date': datetime.date(2019, 1, 1),
                 'product_name': 'NothingMuch',
-                'product_os_platform': 'Ginko',
+                'product_os_platform': 'Ginko\xe2\x98\xa2',
                 'product_os_version': '2.3.2',
                 'product_version': '10.0.5a',
                 'build': 'release',