/
similarity.py
116 lines (101 loc) · 3.71 KB
/
similarity.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
# -*- coding: utf-8 -*-
"""
@author:XuMing(xuming624@qq.com)
@description:
"""
import numpy as np
from text2vec.algorithm.distance import cosine_distance
from text2vec.algorithm.rank_bm25 import BM25Okapi
from text2vec.utils.logger import get_logger
from text2vec.utils.tokenizer import Tokenizer
from text2vec.vector import Vector, EmbType
logger = get_logger(__name__)
class SimType(object):
COSINE = 'cosine'
WMD = 'wmd'
class Similarity(Vector):
def __init__(self, similarity_type=SimType.COSINE,
embedding_type=EmbType.W2V,
sequence_length=128,
w2v_path='',
w2v_kwargs=None,
bert_model_folder='',
bert_layer_nums=4):
"""
Cal text similarity
:param similarity_type:
:param embedding_type:
:param sequence_length:
:param w2v_path:
:param w2v_kwargs:
:param bert_model_folder:
:param bert_layer_nums:
"""
super(Similarity, self).__init__(embedding_type=embedding_type,
w2v_path=w2v_path,
w2v_kwargs=w2v_kwargs,
sequence_length=sequence_length,
bert_model_folder=bert_model_folder,
bert_layer_nums=bert_layer_nums)
self.similarity_type = similarity_type
self.sequence_length = sequence_length
def get_score(self, text1, text2):
"""
Get score between text1 and text2
:param text1: str
:param text2: str
:return: float, score
"""
ret = 0.0
if not text1.strip() or not text2.strip():
return ret
token_1 = self.tokenize(text1)
token_2 = self.tokenize(text2)
if self.similarity_type == SimType.COSINE:
emb_1 = self.encode(token_1)
emb_2 = self.encode(token_2)
ret = cosine_distance(emb_1, emb_2)
elif self.similarity_type == SimType.WMD:
ret = 1. / (1. + self.model.w2v.wmdistance(token_1, token_2))
return ret
class SearchSimilarity(object):
def __init__(self, corpus):
"""
Search sim doc with rank bm25
:param corpus: list of str.
A list of doc.(no need segment, do it in init)
"""
self.corpus = corpus
self.corpus_seg = None
self.bm25_instance = None
self.tokenizer = Tokenizer()
def init(self):
if not self.bm25_instance:
if not self.corpus:
logger.error('corpus is none, set corpus with docs.')
raise ValueError("must set corpus, which is documents, list of str")
if isinstance(self.corpus, str):
self.corpus = [self.corpus]
self.corpus_seg = {k: self.tokenizer.tokenize(k) for k in self.corpus}
self.bm25_instance = BM25Okapi(corpus=list(self.corpus_seg.values()))
def get_similarities(self, query, n=5):
"""
Get similarity between `query` and this docs.
:param query: str
:param n: int, num_best
:return: result, dict, float scores, docs rank
"""
scores = self.get_scores(query)
rank_n = np.argsort(scores)[::-1]
if n > 0:
rank_n = rank_n[:n]
return [self.corpus[i] for i in rank_n]
def get_scores(self, query):
"""
Get scores between query and docs
:param query: input str
:return: numpy array, scores for query between docs
"""
self.init()
tokens = self.tokenizer.tokenize(query)
return self.bm25_instance.get_scores(query=tokens)