hash_string() should not depend on python's internal unicode represen…

…tation, also fixes explosion/sense2vec#5 for py2
explosion · Mar 6, 2016 · b740f20 · b740f20
1 parent 7adbd7a
commit b740f20
Showing 1 changed file with 2 additions and 4 deletions.
diff --git a/spacy/strings.pyx b/spacy/strings.pyx
@@ -23,10 +23,8 @@ import ujson as json
 
 
 cpdef hash_t hash_string(unicode string) except 0:
-    # This has to be like this for
-    chars = <char*>PyUnicode_AS_DATA(string)
-    size = PyUnicode_GET_DATA_SIZE(string)
-    return hash64(chars, size, 1)
+    chars = string.encode('utf8')
+    return hash64(<char*>chars, len(chars), 1)
 
 
 cdef unicode _decode(const Utf8Str* string):