Browse files

[offline-renderer] Remove leading/trailing/duplicate spaces

Signed-off-by: Christopher Hall <hsw@openmoko.com>
  • Loading branch information...
1 parent 0233f7b commit f7a49100bcf00ce0e4abee4746a3a26b5a3c37db @hxw hxw committed Mar 31, 2010
Showing with 17 additions and 6 deletions.
  1. +15 −4 host-tools/offline-renderer/ArticleIndex.py
  2. +2 −2 host-tools/offline-renderer/littleparser.py
View
19 host-tools/offline-renderer/ArticleIndex.py
@@ -541,6 +541,7 @@ def strip_accents(s):
def bigram_encode(title):
global bigram
+ global whitespaces
result = ''
title = strip_accents(title)
@@ -563,7 +564,11 @@ def bigram_encode(title):
result += chr(ord(title[0]))
#else:
# result += '?'
- return result.strip()
+
+ # compact all spaces
+ result = whitespaces.sub(' ', result).strip()
+
+ return result
def output_fnd(filename, article_index, language_processor):
@@ -603,7 +608,12 @@ def output_fnd(filename, article_index, language_processor):
def sort_key(key):
global KEYPAD_KEYS
- return ''.join(c for c in strip_accents(language_processor.translate(key).lower().strip()) if c in KEYPAD_KEYS)
+ global whitepaces
+
+ result = ''.join(c for c in strip_accents(language_processor.translate(key).lower()) if c in KEYPAD_KEYS)
+ # compact all spaces
+ result = whitespaces.sub(' ', result).strip()
+ return result
PrintLog.message(u'Sorting titles')
start_time = time.time()
@@ -713,7 +723,8 @@ def translate(self, text):
class LanguageNull(LanguageProcessor):
"""no-op class"""
def translate(self, text):
- return text
+ """null translation => only strip spaces"""
+ return text.strip()
class Furigana(LanguageProcessor):
@@ -902,7 +913,7 @@ def translate(self, text):
result += r + " "
n = n.next
- return result
+ return result.strip()
# run the program
View
4 host-tools/offline-renderer/littleparser.py
@@ -58,8 +58,8 @@ def translate(self, text):
return unq
if type(self.buffer) == unicode:
- return self.buffer
- return unicode(self.buffer, 'utf-8')
+ return self.buffer.strip()
+ return unicode(self.buffer, 'utf-8').strip()
# tests

0 comments on commit f7a4910

Please sign in to comment.