-
Notifications
You must be signed in to change notification settings - Fork 0
/
politikus_persons.py
62 lines (50 loc) · 1.79 KB
/
politikus_persons.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
import pandas
import requests
from tqdm import tqdm,trange
HEADERS = {'Accept': 'application/json'}
BASE_URL = "https://politikus.sinarproject.org/@search?" \
"portal_type=Person&" \
"fullobjects=1"
# default batch is 25 items
BATCH = 25
total_items = requests.get(BASE_URL,
headers=HEADERS).json()['items_total']
peps = []
for i in tqdm(range(0, total_items, 25), desc="Download Progress"):
b_start = i+1
URL = BASE_URL + "&b_start=" + str(b_start)
r = requests.get(URL, headers=HEADERS)
for person in r.json()['items']:
pep = {}
pep['name'] = person['name']
pep['summary'] = person['summary']
pep['pepStatusDetails'] = person['pepStatusDetails']
pep['birth_date'] = person['birth_date']
if person['gender']:
pep['gender'] = person['gender']['title']
else:
pep['gender'] = None
#tax residencies
taxResidencies = []
if person['taxResidencies']:
for tax_country in person['taxResidencies']:
taxResidencies.append(tax_country['title'])
if taxResidencies:
pep['taxResidencies'] = ",".join(taxResidencies)
else:
pep['taxResidencies'] = None
nationalities = []
if person['nationalities']:
for nationality in person['nationalities']:
nationalities.append(nationality['title'])
if nationalities:
pep['nationalities'] = ",".join(taxResidencies)
else:
pep['nationalities'] = None
#ids
pep['UID'] = person['UID']
pep['rdf_id'] = person['@id']
peps.append(pep)
df = pandas.DataFrame(peps)
print("Exporting politikus-persons.csv in current directory")
df.to_csv('politikus-persons.csv')