-
Notifications
You must be signed in to change notification settings - Fork 1
/
ManageExRankService.py
56 lines (51 loc) · 2.13 KB
/
ManageExRankService.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
#-*- coding:utf-8 -*-
from urllib import response
import urllib.request
import json
import time
from urllib import parse
from sqlalchemy import create_engine
from pandas import json_normalize
from urllib.error import HTTPError
from sqlalchemy.sql.expression import false
engine = create_engine("postgresql://127.0.0.1:5432/postgres?user=postgres&password=1234")
def get_response(request):
try:
response = urllib.request.urlopen(request)
return response
except HTTPError as ex:
for i in range(5):
print("{}차 재시도 전 대기중".format(i+1))
time.sleep(60)
try:
response = urllib.request.urlopen(request)
print("{}차 재시도 성공".format(i+1))
return response
except HTTPError as ex:
print("{}차 재시도 실패".format(i+1))
if(i==4):
return None
def main():
pageNo = 0
maxPage = 10
serviceKey = "key"
while(pageNo < maxPage):
pageNo += 1
url = "http://apis.data.go.kr/1192000/openapi/service/ManageExpRankService/getExpRankList?ServiceKey={}&pageNo={}&numOfRows=10&type=json&baseDt=201510".format(serviceKey, pageNo)
request = urllib.request.Request(url)
response = get_response(request)
rescode = response.getcode()
if(rescode==200):
try:
response_body = json.loads(response.read())
maxPage = response_body['responseJson']['header']['totalCount'] // response_body['responseJson']['header']['numOfRows'] + 1
data_list = response_body['responseJson']['body']['item']
df = json_normalize(data_list)
df.to_sql("ManageExRankService", engine, if_exists='append', index=False, chunksize=1000)
print("{}/{} 페이지 수집 / 적재 성공".format(pageNo, maxPage))
except Exception as e:
print("{}/{} 페이지 수집 / 적재 실패".format(pageNo, maxPage))
else:
print("{}/{} 페이지 수집 / 적재 실패".format(pageNo, maxPage))
if __name__ : "__main__"
main()