Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CHILDESCorpusReader: multiple stems correspond to their source token; keep suffix PoS info #1064

Merged
merged 2 commits into from Sep 1, 2015
Merged

CHILDESCorpusReader: multiple stems correspond to their source token; keep suffix PoS info #1064

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f676b7e
stems from the same word now treated as one word
jacksonllee Aug 3, 2015
d69cb90
glue multiple stems for a word; salvage the lost PoS info of suffixes
jacksonllee Aug 4, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
26 changes: 19 additions & 7 deletions nltk/corpus/reader/childes.py
View file
Open in desktop
Expand Up @@ -270,7 +270,7 @@ def _get_words(self, fileid, speaker, sent, stem, relation, pos,
# select speakers
if speaker == 'ALL' or xmlsent.get('who') in speaker:
for xmlword in xmlsent.findall('.//{%s}w' % NS):
infl = None ; suffixStem = None
infl = None ; suffixStem = None; suffixTag = None
# getting replaced words
if replace and xmlsent.find('.//{%s}w/{%s}replacement'
% (NS,NS)):
Expand Down Expand Up @@ -307,6 +307,8 @@ def _get_words(self, fileid, speaker, sent, stem, relation, pos,
suffixStem = xmlsuffix.text
except AttributeError:
suffixStem = ""
if suffixStem:
word += "~"+suffixStem
# pos
if relation or pos:
try:
Expand All @@ -316,11 +318,22 @@ def _get_words(self, fileid, speaker, sent, stem, relation, pos,
tag = xmlpos[0].text+":"+xmlpos2[0].text
else:
tag = xmlpos[0].text
word = (word,tag)
except (AttributeError,IndexError) as e:
word = (word,None)
if suffixStem:
suffixStem = (suffixStem,None)
tag = ""
try:
xmlsuffixpos = xmlword.findall('.//{%s}mor/{%s}mor-post/{%s}mw/{%s}pos/{%s}c'
% (NS,NS,NS,NS,NS))
xmlsuffixpos2 = xmlword.findall('.//{%s}mor/{%s}mor-post/{%s}mw/{%s}pos/{%s}s'
% (NS,NS,NS,NS,NS))
if xmlsuffixpos2:
suffixTag = xmlsuffixpos[0].text+":"+xmlsuffixpos2[0].text
else:
suffixTag = xmlsuffixpos[0].text
except:
pass
if suffixTag:
tag += "~"+suffixTag
word = (word, tag)
# relational
# the gold standard is stored in
# <mor></mor><mor type="trn"><gra type="grt">
Expand Down Expand Up @@ -357,8 +370,6 @@ def _get_words(self, fileid, speaker, sent, stem, relation, pos,
except:
pass
sents.append(word)
if suffixStem:
sents.append(suffixStem)
if sent or relation:
results.append(sents)
else:
Expand Down Expand Up @@ -480,3 +491,4 @@ def demo(corpus_root=None):

if __name__ == "__main__":
demo()