-
Notifications
You must be signed in to change notification settings - Fork 1
/
regular_expressions.py
94 lines (71 loc) · 2.34 KB
/
regular_expressions.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
# ========================================================================
# Copyright 2023 Emory University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========================================================================
__author__ = 'Jinho D. Choi'
import re
def tokenize(text: str) -> list[str]:
re_tok = re.compile(r'([",.]|\s+|n\'t)')
tokens, prev_idx = [], 0
for m in re_tok.finditer(text):
t = text[prev_idx:m.start()].strip()
if t: tokens.append(t)
t = m.group().strip()
if t:
if tokens and tokens[-1] in {'Mr', 'Ms'} and t == '.':
tokens[-1] = tokens[-1] + t
else:
tokens.append(t)
prev_idx = m.end()
t = text[prev_idx:]
if t: tokens.append(t)
return tokens
if __name__ == '__main__':
# match()
re_mr = re.compile(r'M[rs]\.')
m = re_mr.match('Mr. Wayne')
print(m)
if m: print(m.start(), m.end())
# group()
print(m.groups())
re_mr = re.compile(r'(M[rs])(\.)')
m = re_mr.match('Ms.')
print(m)
print(m.group())
print(m.groups())
print(m.group(0), m.group(1), m.group(2))
m = re_mr.match('Mrs.')
print(m)
# search()
s1 = 'Mr. and Ms. Wayne are here'
print(re_mr.match(s1))
s2 = 'Here are Mr. and Ms. Wayne'
print(re_mr.match(s2))
print(re_mr.search(s1))
print(re_mr.search(s2))
# findall()
print(re_mr.findall(s1))
print(re_mr.findall(s2))
# finditer()
ms = re_mr.finditer(s1)
for m in ms: print(m)
ms = re_mr.finditer(s2)
for m in ms: print(m)
# sub()
print(re_mr.sub('Dr.', 'I met Mr. Wayne and Ms. Kyle.'))
# tokenize()
text = 'Mr. Wayne isn\'t the hero we need, but "the one" we deserve.'
print(tokenize(text))
text = 'Ms. Wayne is "Batgirl" but not "the one".'
print(tokenize(text))