-
Notifications
You must be signed in to change notification settings - Fork 86
/
abook-sentences.py
executable file
·112 lines (81 loc) · 2.67 KB
/
abook-sentences.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2014, 2018 Guenter Bartsch
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#
# extract sentences from audiobook prompts using our pre-trained punkt
# tokenizer
#
import sys
import os
import codecs
import traceback
import logging
import pickle
from optparse import OptionParser
from StringIO import StringIO
from nltools import misc
from nltools.tokenizer import tokenize
PROC_TITLE = 'abook-sentences'
PUNKT_PICKLEFN = 'data/dst/tokenizers/punkt-de.pickle'
CORPORADIR = 'data/dst/text-corpora'
LANG = 'de'
#
# init terminal
#
misc.init_app (PROC_TITLE)
#
# command line
#
parser = OptionParser("usage: %prog [options] foo-1.prompt [foo-2.prompt ...]")
parser.add_option("-n", "--name", dest="name", type="str", default="abook",
help="dataset name (default: abook)")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
help="enable debug output")
(options, args) = parser.parse_args()
if options.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
if len(args) < 1:
parser.print_usage()
sys.exit(1)
outputfn = '%s/%s.txt' % (CORPORADIR, options.name)
#
# load punkt tokenizer
#
logging.info ("loading %s ..." % PUNKT_PICKLEFN)
with open(PUNKT_PICKLEFN, mode='rb') as f:
punkt = pickle.load(f)
#
# main
#
logging.info ("extracting sentences ...")
cnt = 0
with codecs.open(outputfn, 'w', 'utf8') as outputf:
for inputfn in args:
with codecs.open(inputfn, 'r', 'utf8') as inputf:
txt = inputf.read()
sentences = punkt.tokenize(txt, realign_boundaries=True)
for sentence in sentences:
s = (u' '.join(tokenize(sentence, lang=LANG))).strip()
if not s:
continue
logging.debug("sentence: %s" % s)
outputf.write('%s\n' % s)
cnt += 1
logging.info ('%s written, %d sentences.' % (outputfn, cnt))