[Fixes #4357] tags do not support unicode text

geonode/base/forms.py

@@ -17,7 +17,6 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 #########################################################################
-
 from fields import MultiThesauriField
 from widgets import MultiThesauriWidget
 
@@ -112,14 +111,14 @@ def render(self, name, value, attrs=None):
         if isinstance(value, basestring):
             vals = value
         elif value:
-            vals = ','.join([str(i.tag.name) for i in value])
+            vals = ','.join([i.tag.name for i in value])
         else:
             vals = ""
         output = ["""<div class="keywords-container"><span class="input-group">
-                <input class='form-control'
-                       id='id_resource-keywords'
-                       name='resource-keywords'
-                       value='%s'><br/>""" % (vals)]
+                <input class="form-control"
+                       id="id_resource-keywords"
+                       name="resource-keywords"
+                       value="%s"><br/>""" % (vals)]
         output.append(
             '<div id="treeview" class="" style="display: none"></div>')
         output.append(
@@ -420,11 +419,12 @@ def unicode_escape(unistr):
         keywords = self.cleaned_data['keywords']
         _unsescaped_kwds = []
         for k in keywords:
-            _k = urllib.unquote((u'%s' % k).encode('utf-8')).split(",")
+            _k = urllib.unquote(('%s' % k)).split(",")
             if not isinstance(_k, basestring):
                 for _kk in [x.strip() for x in _k]:
                     _kk = HTMLParser.HTMLParser().unescape(unicode_escape(_kk))
-                    # _hk = HierarchicalKeyword.objects.extra(where=["%s LIKE name||'%%'"], params=[_kk])
+                    # Simulate JS Unescape
+                    _kk = _kk.replace('%u', r'\u').decode('unicode-escape') if '%u' in _kk else _kk
                     _hk = HierarchicalKeyword.objects.filter(name__contains='%s' % _kk.strip())
                     if _hk and len(_hk) > 0:
                         _unsescaped_kwds.append(_hk[0])

geonode/base/models.py

@@ -903,13 +903,13 @@ def metadata_completeness(self):
         return '{}%'.format(len(filled_fields) * 100 / len(required_fields))
 
     def keyword_list(self):
-        return [kw.name.encode("utf-8", "replace") for kw in self.keywords.all()]
+        return [kw.name for kw in self.keywords.all()]
 
     def keyword_slug_list(self):
-        return [kw.slug.encode("utf-8", "replace") for kw in self.keywords.all()]
+        return [kw.slug for kw in self.keywords.all()]
 
     def region_name_list(self):
-        return [region.name.encode("utf-8", "replace") for region in self.regions.all()]
+        return [region.name for region in self.regions.all()]
 
     def spatial_representation_type_string(self):
         if hasattr(self.spatial_representation_type, 'identifier'):

geonode/geoserver/signals.py

@@ -293,7 +293,7 @@ def geoserver_post_save_local(instance, *args, **kwargs):
 
     if any(instance.keyword_list()):
         keywords = instance.keyword_list()
-        gs_resource.keywords = [kw.decode("utf-8", "replace") for kw in list(set(keywords))]
+        gs_resource.keywords = [kw for kw in list(set(keywords))]
 
         # gs_resource should only be called if
         # ogc_server_settings.BACKEND_WRITE_ENABLED == True

geonode/layers/tests.py

@@ -249,6 +249,36 @@ def test_layer_save(self):
             lyr.keyword_list(), [
                 u'here', u'keywords', u'populartag', u'saving'])
 
+        # Test exotic encoding Keywords
+        lyr.keywords.add(*[u'論語', u'ä', u'ö', u'ü', u'ß'])
+        lyr.save()
+        self.assertEqual(
+            lyr.keyword_list(), [
+                u'here', u'keywords', u'populartag', u'saving',
+                u'ß', u'ä', u'ö', u'ü', u'論語'])
+
+        self.client.login(username='admin', password='admin')
+        response = self.client.get(reverse('layer_detail', args=(lyr.alternate,)))
+        self.failUnlessEqual(response.status_code, 200)
+
+        response = self.client.get(reverse('layer_metadata', args=(lyr.alternate,)))
+        self.failUnlessEqual(response.status_code, 200)
+
+        from geonode.base.models import HierarchicalKeyword as hk
+        keywords = hk.dump_bulk_tree()
+        self.assertEqual(keywords, [
+            {"text": u"here", "href": "here", "id": 2},
+            {"text": u"keywords", "href": "keywords", "id": 4},
+            {"text": u"layertagunique", "href": "layertagunique", "id": 3},
+            {"text": u"populartag", "href": "populartag", "id": 1},
+            {"text": u"saving", "href": "saving", "id": 5},
+            {"text": u"ß", "href": "ss", "id": 9},
+            {"text": u"ä", "href": "a", "id": 10},
+            {"text": u"ö", "href": "o", "id": 7},
+            {"text": u"ü", "href": "u", "id": 8},
+            {"text": u"論語", "href": "lun-yu", "id": 6}
+        ])
+
     def test_layer_links(self):
         lyr = Layer.objects.filter(storeType="dataStore").first()
         self.assertEquals(lyr.storeType, "dataStore")

geonode/static/geonode/js/search/search.js

@@ -376,7 +376,7 @@
         }, true);
     }
 
-    // Hyerarchical keywords listeners
+    // Hierarchical keyword listeners
     $scope.$on('select_h_keyword', function($event, element){
       var data_filter = 'keywords__slug__in';
       var query_entry = [];

geonode/templates/metadata_form_js.html

@@ -526,11 +526,11 @@
                 var formData = $(this).serializeArray();
                 formData.forEach(function(entry) {
                     try{
-                        if((typeof entry["name"]) === "string" && 
+                        if((typeof entry["name"]) === "string" &&
                             entry["name"].includes("keywords")) {
                             entry["value"] = escape(entry["value"]);
                         }
-                        else if((typeof entry["name"]) !== "string" && 
+                        else if((typeof entry["name"]) !== "string" &&
                             entry["name"].contains("keywords")) {
                             entry["value"] = escape(entry["value"]);
                         }