/
imports.py
72 lines (55 loc) · 1.64 KB
/
imports.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
import logging
import os
import warnings
from ...imports import SUPPRESS_DEP_WARNINGS
# os.environ['DISABLE_V2_BEHAVIOR'] = '1'
if SUPPRESS_DEP_WARNINGS:
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
logging.getLogger("tensorflow").setLevel(logging.ERROR)
logging.getLogger("tensorflow_hub").setLevel(logging.ERROR)
warnings.simplefilter(action="ignore", category=FutureWarning)
try:
import tensorflow as tf
TF_INSTALLED = True
except ImportError:
TF_INSTALLED = False
if TF_INSTALLED:
tf.autograph.set_verbosity(1)
import os.path
import re
import string
import numpy as np
from scipy.sparse import coo_matrix, spmatrix
from sklearn.base import BaseEstimator
try: # sklearn<0.24.x
from sklearn.linear_model.base import LinearClassifierMixin, SparseCoefMixin
except ImportError: # sklearn>=0.24.x
from sklearn.linear_model._base import LinearClassifierMixin, SparseCoefMixin
import syntok.segmenter as segmenter
from joblib import dump, load
from sklearn.datasets import load_files
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder
from sklearn.svm import LinearSVC
# ktrain imported locally in ner.py
# import ktrain
# pandas imported locally in classifier.py
# import pandas as pd
try:
import langdetect
LANGDETECT = True
except:
LANGDETECT = False
try:
import cchardet as chardet
CHARDET = True
except:
CHARDET = False
try:
import jieba
JIEBA = True
except:
JIEBA = False