clean code with test passed

textmulclassify/data_structures/__init__.py

@@ -1,2 +1,4 @@
+__all__ = ["Distribution", "TMCModel", "TMCTree"]
+
 from .model import Distribution, TMCModel
 from .tree import TMCTree

textmulclassify/data_structures/model.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from etl_utils import UnicodeUtils, process_notifier, cpickle_cache
-from collections import Counter, defaultdict
+from collections import Counter
 from urwid import is_wide_char
 import random
 
@@ -51,7 +51,7 @@ def tags_model__append_more_features_when_recommend(cls, item, sorted_features):
 
     @classmethod
     def is_valid_tag(cls, tag1):
-        return self.tags_tree.has_node(tag1)
+        return cls.tags_tree.has_node(tag1)
 
     def __repr__(self):
         print "item_id", self.item_id, "\n"

textmulclassify/data_structures/tree.py

@@ -2,10 +2,9 @@
 
 from etl_utils import UnicodeUtils, process_notifier, uprint, cached_property, slots_with_pickle
 from collections import defaultdict, Counter
+from ..lib.read_manual_kps import ReadManualKps
 
 
-from bunch import Bunch
-
 # 使用 __slots__ 属性 降低内存使用
 # 优化例子: 初高中物理数学内存从 5.6G 降低到 5.0G.
 
@@ -88,7 +87,7 @@ def add_to_current_tree(current_tree, parent):
             add_to_current_tree(self[TMCTree.root_node], TMCTree.root_node)
 
         def import_from_file(file1):
-# import_from_file 暂不支持depth
+            # NOTE import_from_file 暂不支持depth
             for line in UnicodeUtils.read(file1).strip().split(line_split):
                 line = line.strip()
                 if TMCTree.root_node not in self:
@@ -257,11 +256,14 @@ def filter_valid_tags(self, tags):
 
     @cached_property
     def total_nodes(self):
-        return set([node1 for f1, nodes in feature_to_nodes.iteritems() for node1 in nodes])
+        return set([node1 for f1, nodes in self.feature_to_nodes.iteritems() for node1 in nodes])
 
     def rich_train_data_by_editor(self, files=[]):
         """ 通过人工编辑规则增强Train Data """
         # 20140910_1427 没效果，反而有一两个百分点下降。
+        import jieba
+        dict_dir = None
+
         for file1 in files:
             parsed = ReadManualKps.process(dict_dir + file1)
             for node_name1, node_features in parsed.iteritems():

textmulclassify/lib/evaluate.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 
-from ..data_structures.tree import TMCTree
 from etl_utils import uprint
 
 
@@ -13,6 +12,7 @@ def __init__(self, tags_tree, items):
             i1['eval_result'] = []
 
         # 验证数据结构
+        from ..data_structures.tree import TMCTree  # load lazily
         assert isinstance(tags_tree, TMCTree)  # name TODO
         assert isinstance(self.items, list)
         assert 'original_tags' in self.items[0]
@@ -48,8 +48,8 @@ def update(obj, method, num):
             total_counts.recommend += len(recommend_tags)
 
             # processed_* 只是为了处理 epcp 依赖的顺序，即前面处理了，后面就没机会了
-            processed_original_tags   = set([])
-            processed_recommend_tags  = set([])
+            # processed_original_tags   = set([])
+            # processed_recommend_tags  = set([])
 
             def func(counts, is_precision=False):
                 if not is_precision:

textmulclassify/lib/features_weight.py

@@ -4,9 +4,8 @@
 # refactored by @mvj3
 
 import os
-import sys
 import json
-from etl_utils import process_notifier, UnicodeUtils, uprint, cached_property, singleton
+from etl_utils import UnicodeUtils, cached_property, singleton
 from collections import defaultdict, Counter
 import jieba.posseg as posseg
 
@@ -84,7 +83,7 @@ def stop_words_set(self):
         return set([w1.strip() for file1 in self.classify.stop_words_files
                     for w1 in UnicodeUtils.read(file1).split("\n")])
 
-    #@profile
+    # @profile
     def extract_feature_words(self, in_text):
         """ 专业词汇抽取 + 对长词(3)再做分词 """
         assert isinstance(in_text, unicode), in_text

textmulclassify/lib/similarity.py

@@ -3,8 +3,8 @@
 # original author is @junchen and @LiHeng
 # refactored by @mvj3
 
-#from scipy.spatial.distance import cosine as scipy_cosine
-#from collections import OrderedDict
+# from scipy.spatial.distance import cosine as scipy_cosine
+# from collections import OrderedDict
 from math import sqrt