🐛 Fixing labeling dict bug, added PAD token to seq label dict.

BrikerMan · May 22, 2019 · d2af50c · d2af50c
1 parent cc6d4ea
commit d2af50c
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/kashgari/pre_processors/labeling_processor.py b/kashgari/pre_processors/labeling_processor.py
@@ -35,7 +35,9 @@ def _build_label_dict(self,
         Args:
             label_list: corpus label list
         """
-        label2idx: Dict[str: int] = {}
+        label2idx: Dict[str: int] = {
+            self.token_pad: 0
+        }
 
         token2count = {}
         for label_set in label_list:

diff --git a/kashgari/tasks/labeling/base_model.py b/kashgari/tasks/labeling/base_model.py
@@ -77,13 +77,13 @@ def compile_model(self, **kwargs):
             weight = np.full((len(idx2label),), 50)
             for idx, label in idx2label.items():
                 if label == self.embedding.processor.token_pad:
-                    weight[idx] = 0
+                    weight[idx] = 0.1
                 if label in ['O']:
                     weight[idx] = 1
             weight_dict = {}
             for idx, label in idx2label.items():
                 weight_dict[label] = weight[idx]
-                logging.debug(f"label weights set to {weight_dict}")
+            logging.debug(f"label weights set to {weight_dict}")
             kwargs['loss'] = weighted_categorical_crossentropy(weight)
         super(BaseLabelingModel, self).compile_model(**kwargs)
 
@@ -125,6 +125,7 @@ def build_model_arc(self):
 
 
 if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
     from kashgari.tasks.labeling import CNNLSTMModel
     from kashgari.corpus import ChineseDailyNerCorpus