forked from huggingface/datasets
-
Notifications
You must be signed in to change notification settings - Fork 0
/
blog_authorship_corpus.py
168 lines (141 loc) 路 6.79 KB
/
blog_authorship_corpus.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
from __future__ import absolute_import, division, print_function
import glob
import logging
import os
import datasets
_CITATION = """\
@inproceedings{schler2006effects,
title={Effects of age and gender on blogging.},
author={Schler, Jonathan and Koppel, Moshe and Argamon, Shlomo and Pennebaker, James W},
booktitle={AAAI spring symposium: Computational approaches to analyzing weblogs},
volume={6},
pages={199--205},
year={2006}
}
"""
_DESCRIPTION = """\
The Blog Authorship Corpus consists of the collected posts of 19,320 bloggers gathered from blogger.com in August 2004. The corpus incorporates a total of 681,288 posts and over 140 million words - or approximately 35 posts and 7250 words per person.
Each blog is presented as a separate file, the name of which indicates a blogger id# and the blogger鈥檚 self-provided gender, age, industry and astrological sign. (All are labeled for gender and age but for many, industry and/or sign is marked as unknown.)
All bloggers included in the corpus fall into one of three age groups:
路 8240 "10s" blogs (ages 13-17),
路 8086 "20s" blogs(ages 23-27)
路 2994 "30s" blogs (ages 33-47).
For each age group there are an equal number of male and female bloggers.
Each blog in the corpus includes at least 200 occurrences of common English words. All formatting has been stripped with two exceptions. Individual posts within a single blogger are separated by the date of the following post and links within a post are denoted by the label urllink.
The corpus may be freely used for non-commercial research purposes
"""
_URL = "https://u.cs.biu.ac.il/~koppel/BlogCorpus.htm"
_DATA_URL = "http://www.cs.biu.ac.il/~koppel/blogs/blogs.zip"
class BlogAuthorshipCorpusConfig(datasets.BuilderConfig):
"""BuilderConfig for BlogAuthorship."""
def __init__(self, data_url, **kwargs):
"""BuilderConfig for BlogAuthorship
Args:
data_url: `string`, url to the dataset (word or raw level)
**kwargs: keyword arguments forwarded to super.
"""
super(BlogAuthorshipCorpusConfig, self).__init__(
version=datasets.Version(
"1.0.0",
),
**kwargs,
)
self.data_url = data_url
class BlogAuthorshipCorpus(datasets.GeneratorBasedBuilder):
"""TODO(BlogAuthorship): Short description of my dataset."""
VERSION = datasets.Version("0.1.0")
BUILDER_CONFIGS = [
BlogAuthorshipCorpusConfig(
name="blog-authorship-corpus",
data_url=_DATA_URL,
description="word level dataset. No processing is needed other than replacing newlines with <eos> tokens.",
),
]
def _info(self):
return datasets.DatasetInfo(
# This is the description that will appear on the datasets page.
description=_DESCRIPTION,
# datasets.features.FeatureConnectors
features=datasets.Features(
{
"text": datasets.Value("string"),
"date": datasets.Value("string"),
"gender": datasets.Value("string"),
"age": datasets.Value("int32"),
"horoscope": datasets.Value("string"),
"job": datasets.Value("string"),
}
),
# If there's a common (input, target) tuple from the features,
# specify them here. They'll be used if as_supervised=True in
# builder.as_dataset.
supervised_keys=None,
# Homepage of the dataset for documentation
homepage=_URL,
citation=_CITATION,
)
def _split_generators(self, dl_manager):
"""Returns SplitGenerators."""
if self.config.name == "blog-authorship-corpus":
data = dl_manager.download_and_extract(self.config.data_url)
data_dir = os.path.join(data, "blogs")
files = sorted(glob.glob(os.path.join(data_dir, "*.xml")))
train_files = []
validation_files = []
for i, file_path in enumerate(files):
# 95% / 5% (train / val) split
if i % 20 == 0:
validation_files.append(file_path)
else:
train_files.append(file_path)
return [
datasets.SplitGenerator(
name=datasets.Split.TRAIN,
gen_kwargs={"files": train_files, "split": "train"},
),
datasets.SplitGenerator(
name=datasets.Split.VALIDATION,
gen_kwargs={"files": validation_files, "split": "validation"},
),
]
else:
raise ValueError("{} does not exist".format(self.config.name))
def _generate_examples(self, files, split):
def parse_date(line):
# parse line to date
return line.strip().split("<date>")[-1].split("</date>")[0]
for file_path in files:
counter = 0
file_name = os.path.basename(file_path)
logging.info("generating examples from = %s", file_path)
file_id, gender, age, job, horoscope = tuple(file_name.split(".")[:-1])
# Note: import xml.etree.ElementTree as etree does not work. File cannot be parsed
# use open instead
with open(file_path, encoding="utf-8") as f:
# some files are corrupted, so have to work with python`s try here
try:
date = ""
for line in f:
line = line.strip()
if "<date>" in line:
date = parse_date(line)
elif line != "" and not line.startswith("<"):
# need sub_id to be certain that no tf_records is identical
sub_id = counter
counter += 1
if date == "":
logging.warning("Date missing for {} in {}".format(line, file_name))
assert date is not None, "Date is missing before {}".format(line)
blog = {
"text": line,
"date": date,
"gender": gender,
"age": int(age),
"job": job,
"horoscope": horoscope,
}
yield "{}_{}_{}".format(file_id, sub_id, date), blog
else:
continue
except UnicodeDecodeError as e:
logging.warning("{} cannot be loaded. Error message: {}".format(file_path, e))