Fix UnicodeDecodeError on creating items with tags including extended…

… characters

CHANGES.rst

@@ -4,7 +4,8 @@ Changelog
 1.1.2 (unreleased)
 ^^^^^^^^^^^^^^^^^^
 
-- Nothing changed yet.
+- Corrige ``UnicodeDecodeError`` ao criar itens com tags que possuem caracteres estendidos (fecha `brasil.gov.portal#288 <https://github.com/plonegovbr/brasil.gov.portal/issues/288>`_).
+  [hvelarde]
 
 
 1.1.1 (2017-11-08)

src/brasil/gov/agenda/content/agenda.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from brasil.gov.agenda.interfaces import IAgenda
 from plone.dexterity.content import Container
+from plone.dexterity.utils import safe_utf8
 from plone.indexer.decorator import indexer
 from zope.interface import implementer
 
@@ -12,7 +13,7 @@ class Agenda(Container):
 
 @indexer(IAgenda)
 def tags(obj):
-    """Indexa tags de Agenda
-    """
-    tags = obj.subjects
-    return tags
+    """Indexa tags de Agenda."""
+    if obj.subjects is None:
+        return ()
+    return tuple(safe_utf8(s) for s in obj.subjects)

src/brasil/gov/agenda/content/agendadiaria.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-
 from brasil.gov.agenda import _
 from brasil.gov.agenda import utils
 from brasil.gov.agenda.config import AGENDADIARIAFMT
@@ -9,6 +8,7 @@
 from DateTime import DateTime
 from plone import api
 from plone.dexterity.content import Container
+from plone.dexterity.utils import safe_utf8
 from plone.indexer.decorator import indexer
 from plone.supermodel.interfaces import IDefaultFactory
 from Products.CMFPlone.utils import safe_hasattr
@@ -112,10 +112,10 @@ def validate(self, value):
 
 @indexer(IAgendaDiaria)
 def tags(obj):
-    """Indexa tags de AgendaDiaria
-    """
-    tags = obj.subjects
-    return tags
+    """Indexa tags de AgendaDiaria."""
+    if obj.subjects is None:
+        return ()
+    return tuple(safe_utf8(s) for s in obj.subjects)
 
 
 @indexer(IAgendaDiaria)

src/brasil/gov/agenda/content/compromisso.py

@@ -3,6 +3,7 @@
 from brasil.gov.agenda.interfaces import ICompromisso
 from DateTime import DateTime
 from plone.dexterity.content import Container
+from plone.dexterity.utils import safe_utf8
 from plone.indexer.decorator import indexer
 from Products.CMFPlone.utils import safe_hasattr
 from zope.interface import implementer
@@ -88,10 +89,10 @@ def end_date(obj):
 
 @indexer(ICompromisso)
 def tags(obj):
-    """Indexa tags de Compromisso
-    """
-    tags = obj.subjects
-    return tags
+    """Indexa tags de Compromisso."""
+    if obj.subjects is None:
+        return ()
+    return tuple(safe_utf8(s) for s in obj.subjects)
 
 
 @indexer(ICompromisso)

src/brasil/gov/agenda/content/schema/Agenda.xml

@@ -25,6 +25,8 @@
     </field>
     <field name="subjects" type="zope.schema.Tuple" indexer:searchable="true">
       <description i18n:translate="">Tags são utilizadas para organização de conteúdo</description>
+      <default_value>()</default_value>
+      <missing_value>()</missing_value>
       <required>False</required>
       <title>Tags</title>
       <value_type type="zope.schema.TextLine">

src/brasil/gov/agenda/content/schema/AgendaDiaria.xml

@@ -30,6 +30,8 @@
     <field name="subjects" type="zope.schema.Tuple" indexer:searchable="true">
       <description i18n:translate="">Tags são utilizadas para organização de conteúdo</description>
       <defaultFactory>brasil.gov.agenda.content.agendadiaria.default_subjects</defaultFactory>
+      <default_value>()</default_value>
+      <missing_value>()</missing_value>
       <required>False</required>
       <title>Tags</title>
       <value_type type="zope.schema.TextLine">

src/brasil/gov/agenda/content/schema/Compromisso.xml

@@ -42,6 +42,8 @@
       </field>
     <field name="subjects" type="zope.schema.Tuple" indexer:searchable="true">
       <description i18n:translate="">Tags são utilizadas para organização de conteúdo</description>
+      <default_value>()</default_value>
+      <missing_value>()</missing_value>
       <required>False</required>
       <defaultFactory>brasil.gov.agenda.content.compromisso.default_subjects</defaultFactory>
       <title>Tags</title>

src/brasil/gov/agenda/tests/test_agenda.py

@@ -63,13 +63,13 @@ def test_exclude_from_nav_default(self):
 
     def test_subjects_catalog(self):
         agenda = self.agenda
-        agenda.subjects = ('Brasil', 'Governo')
+        agenda.subjects = (u'Brasil', u'México')
         agenda.reindexObject(idxs=['Subject'])
         ct = self.portal.portal_catalog
         results = ct.searchResults(portal_type='Agenda')
         b = results[0]
         self.assertIn('Brasil', b.Subject)
-        self.assertIn('Governo', b.Subject)
+        self.assertIn('México', b.Subject)
 
     def test_agendadiaria_ordering(self):
         # Create two AgendaDiaria objects

src/brasil/gov/agenda/tests/test_agendadiaria.py

@@ -79,23 +79,21 @@ def test_exclude_from_nav_behavior(self):
 
     def test_subjects_catalog(self):
         agendadiaria = self.agendadiaria
-        agendadiaria.subjects = ('Brasil', 'Governo')
+        agendadiaria.subjects = (u'Brasil', u'México')
         agendadiaria.reindexObject(idxs=['Subject'])
         ct = self.portal.portal_catalog
         results = ct.searchResults(portal_type='AgendaDiaria')
         b = results[0]
         self.assertIn('Brasil', b.Subject)
-        self.assertIn('Governo', b.Subject)
+        self.assertIn('México', b.Subject)
 
     def test_default_subjects(self):
         from brasil.gov.agenda.content.agendadiaria import default_subjects
         agenda = self.agenda
-        agenda.subjects = ('Plone', )
-        # default_factory eh executado no container
-        self.assertIn(
-            'Plone',
-            default_subjects(agenda),
-        )
+        agenda.subjects = (u'Brasil', u'México')
+        # default_factory é executado no container
+        self.assertIn(u'Brasil', default_subjects(agenda))
+        self.assertIn(u'México', default_subjects(agenda))
 
     def test_datevalidator(self):
         from brasil.gov.agenda.content.agendadiaria import DateValidator

src/brasil/gov/agenda/tests/test_compromisso.py

@@ -79,23 +79,21 @@ def test_exclude_from_nav_behavior(self):
 
     def test_subjects_catalog(self):
         compromisso = self.compromisso
-        compromisso.subjects = ('Brasil', 'Governo')
+        compromisso.subjects = (u'Brasil', u'México')
         compromisso.reindexObject(idxs=['Subject'])
         ct = self.portal.portal_catalog
         results = ct.searchResults(portal_type='Compromisso')
         b = results[0]
         self.assertIn('Brasil', b.Subject)
-        self.assertIn('Governo', b.Subject)
+        self.assertIn('México', b.Subject)
 
     def test_default_subjects(self):
         from brasil.gov.agenda.content.compromisso import default_subjects
         agendadiaria = self.agendadiaria
-        agendadiaria.subjects = ('Plone',)
-        # default_factory eh executado no container
-        self.assertIn(
-            'Plone',
-            default_subjects(agendadiaria),
-        )
+        agendadiaria.subjects = (u'Brasil', u'México')
+        # default_factory é executado no container
+        self.assertIn(u'Brasil', default_subjects(agendadiaria))
+        self.assertIn(u'México', default_subjects(agendadiaria))
 
     def test_default_start_date(self):
         from brasil.gov.agenda.content.compromisso import default_start_date