-
Notifications
You must be signed in to change notification settings - Fork 0
/
P3_analyze_headers.py
49 lines (40 loc) · 1.7 KB
/
P3_analyze_headers.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
import pandas as pd
f_save = "data/collated_header_cleaned_dataset.csv"
n_gram = 4
# Starting text to remove, found by inspection from running this code for various n-grams
filtered_lines = [
"? ",
"[unreadable] ",
"description (adapted from applicant's abstract):",
"description (adapted from applicant's abstract)",
"description (provided by applicant):",
"description (provided by applicant)",
"description (provided by the applicant):",
"description (provided by the applicant)",
"description: (provided by applicant)",
"description (adapted from the applicant's abstract):",
"description: (adapted from the investigator's abstract)",
"description: (adapted from the applicant's abstract)",
"description: (adapted from the investigator's abstract)",
"description: (adapted from investigator's abstract)",
"description: (adapted from applicant's abstract)",
"description (adapted from investigator's abstract):",
"description (adapted from the investigator's abstract):",
"description (adapted from the applicant's abstract)",
"project summary / abstract",
"project summary (see instructions):",
]
# Filter the actual header data here
df = pd.read_csv("data/collated_raw_dataset.csv")
for line in filtered_lines:
idx = df["ABSTRACT_TEXT"].str.lower().str.startswith(line)
df.loc[idx, "ABSTRACT_TEXT"] = (
df.loc[idx, "ABSTRACT_TEXT"].str[len(line) :].str.strip()
)
print(line, idx.sum())
# Uncomment to see what's left (v. memory intensive!)
# df['header'] = df.ABSTRACT_TEXT.str.split().str[:n_gram].str.join(' ').str.lower()
# print()
# print(df.header.value_counts()[:20])
# Save what's left
df.to_csv(f_save, index=False)