-
Notifications
You must be signed in to change notification settings - Fork 1
/
data_check.py
73 lines (59 loc) · 2.3 KB
/
data_check.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
import json
import hashlib
import re
def hash_string(s):
return hashlib.sha256(s.encode()).hexdigest()
def match_and_replace_keywords_in_text(text, json_file):
with open(json_file, 'r', encoding='utf-8') as f:
data = json.load(f)
matched_keywords = []
replaced_text = text
i = 0
while i < len(replaced_text):
for j in range(i + 1, len(replaced_text) + 1):
keyword = replaced_text[i:j]
first_char_hash = hash_string(keyword[0])
if first_char_hash not in data:
break
last_char_hash = hash_string(keyword[-1])
keyword_hash = hash_string(keyword)
for item in data[first_char_hash]:
if last_char_hash in item and item[last_char_hash] == keyword_hash:
matched_keywords.append(keyword)
replaced_text = replaced_text.replace(keyword, '***', 1)
i = j - 1
break
else:
continue
break
i += 1
# Replace consecutive "***" with a single "***"
replaced_text = re.sub(r'\*{3,}', '***', replaced_text)
return matched_keywords, replaced_text
def match_and_replace_sensitive_in_text(text):
with open('sensitive_data/json/tencent_sensitive_words.json', 'r', encoding='utf-8') as f:
data = json.load(f)
matched_keywords = []
replaced_text = text
i = 0
while i < len(replaced_text):
for j in range(i + 1, len(replaced_text) + 1):
keyword = replaced_text[i:j]
first_char_hash = hash_string(keyword[0])
if first_char_hash not in data:
break
last_char_hash = hash_string(keyword[-1])
keyword_hash = hash_string(keyword)
for item in data[first_char_hash]:
if last_char_hash in item and item[last_char_hash] == keyword_hash:
matched_keywords.append(keyword)
replaced_text = replaced_text.replace(keyword, '***', 1)
i = j - 1
break
else:
continue
break
i += 1
# Replace consecutive "***" with a single "***"
replaced_text = re.sub(r'\*{3,}', '***', replaced_text)
return matched_keywords, replaced_text