Fix/issue 1 (#2)

* Sort dictionary

* Remove some words

* Add prefix 'ke'

* Fix suffixes visitor

* Add test words

bahasa/data/kamus.txt

@@ -2634,7 +2634,6 @@ beku
 bekuk
 bekuku
 bekukung
-bel
 bela
 belabas
 belacak
@@ -8404,7 +8403,6 @@ geplak
 gepok
 geprak
 gepuk
-gera
 gerabah
 gerabak
 gerabang
@@ -11770,7 +11768,6 @@ kamsia
 kamu
 kamuflase
 kamus
-kan
 kana
 kanaah
 kanaat
@@ -18615,10 +18612,6 @@ nasib
 nasihat
 nasion
 nasional
-nasionalis
-nasionalisasi
-nasionalisme
-nasionalistis
 nasionisme
 naskah
 nasofaring
@@ -22493,7 +22486,6 @@ randi
 randu
 randuk
 randung
-rang
 rangah
 rangai
 rangak

bahasa/stemmer/context.py

@@ -51,7 +51,8 @@ def precedence(self, word):
             match_affix(word, 'me', 'i'),
             match_affix(word, 'di', 'i'),
             match_affix(word, 'pe', 'i'),
-            match_affix(word, 'ter', 'i')
+            match_affix(word, 'ter', 'i'),
+            match_affix(word, suffix='is'),
         ])
 
     def start_stemming_process(self):
@@ -101,7 +102,7 @@ def remove_prefixes(self):
                 return
 
             # Split compound words
-            matches = re.match(r'^(ter|di|me|pe|ber)(.*)(i|kan|an|lah)$', self.original_word)
+            matches = re.match(r'^(ter|di|me|pe|ke|ber)(.*)(i|kan|an|lah)$', self.original_word)
             if matches:
                 counter = 1
                 for _ in self.current_word:

bahasa/stemmer/utils.py

@@ -16,8 +16,10 @@ def load_dictionary(dictionary='default'):
         dictionary = join(base_dir, 'data', 'kamus.txt')
 
     with open(dictionary) as dictionary_file:
-        for data in dictionary_file:
-            word_sets.add(data.strip())
+        words = dictionary_file.read().splitlines()
+        words.sort(key=len, reverse=True)
+        word_sets = set(words)
+
     return word_sets
 
 

bahasa/stemmer/visitor/suffixes.py

@@ -12,7 +12,7 @@ class RemoveInflectionalParticle(object):
     def visit(self, context):
         result = self.remove(context.current_word)
         if result != context.current_word:
-            removed_part = remove_suffix(context.current_word, result)
+            removed_part = context.current_word[len(result):]
             removal = Removal(self, context.current_word, result,
                               removed_part, 'P')
             context.add_removal(removal)
@@ -34,7 +34,7 @@ class RemoveDerivationalSuffix(object):
     def visit(self, context):
         result = self.remove(context.current_word)
         if result != context.current_word:
-            removed_part = remove_suffix(context.current_word, result)
+            removed_part = context.current_word[len(result):]
             removal = Removal(self, context.current_word, result,
                               removed_part, 'DS')
             context.add_removal(removal)
@@ -58,7 +58,7 @@ class RemoveInflectionalPossessivePronoun(object):
     def visit(self, context):
         result = self.remove(context.current_word)
         if result != context.current_word:
-            removed_part = remove_suffix(context.current_word, result)
+            removed_part = context.current_word[len(result):]
             removal = Removal(self, context.current_word, result,
                               removed_part, 'PP')
             context.add_removal(removal)

tests/stemmer/test_stemmer.py

@@ -14,6 +14,14 @@ def test_stem_word(self):
         self.assertEqual(self.stemmer.stem_word('penghancurleburan'), 'hancur lebur')
         self.assertEqual(self.stemmer.stem_word('dilipatgandakan'), 'lipat ganda')
         self.assertEqual(self.stemmer.stem_word('pertanggungjawaban'), 'tanggung jawab')
+        self.assertEqual(self.stemmer.stem_word('ketidakpercayaan'), 'tidak percaya')
+        self.assertEqual(self.stemmer.stem_word('dikuranginya'), 'kurang')
+        self.assertEqual(self.stemmer.stem_word('menyinari'), 'sinar')
+        self.assertEqual(self.stemmer.stem_word('dibelinya'), 'beli')
+        self.assertEqual(self.stemmer.stem_word('gerakan'), 'gerak')
+        self.assertEqual(self.stemmer.stem_word('menangis'), 'tangis')
+        self.assertEqual(self.stemmer.stem_word('perangi'), 'perang')
+        self.assertEqual(self.stemmer.stem_word('nasionalis'), 'nasional')
 
     def test_stem_sentences(self):
         self.assertEqual(