-
Notifications
You must be signed in to change notification settings - Fork 902
/
occtonet.py
248 lines (208 loc) · 8.74 KB
/
occtonet.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
#!/usr/bin/env python3
# coding=utf-8
from datetime import datetime
from io import StringIO
from logging import Logger, getLogger
from typing import Dict, List, Optional
import arrow
import pandas as pd
from requests import Session, cookies
from .lib.exceptions import ParserException
# Abbreviations:
# JP-HKD : Hokkaido
# JP-TH : Tohoku (incl. Niigata)
# JP-TK : Tokyo area (Kanto)
# JP-CB : Chubu
# JP-HR : Hokuriku
# JP-KN : Kansai
# JP-CG : Chugoku
# JP-SK : Shikoku
# JP-KY : Kyushu
# JP-ON : Okinawa
# In selector, they correspond to (format: Japanese original name [english translation])
EXCHANGE_MAPPING = {
"JP-HKD->JP-TH": [
1
], # 北海道・本州間電力連系設備 [Hokkaido-Honshu Electric Power Interconnection Facility]
"JP-TH->JP-TK": [2], # 相馬双葉幹線 [Soma Futaba Trunk Line]
"JP-CB->JP-TK": [3], # 周波数変換設備 [Frequency conversion equipment]
"JP-CB->JP-KN": [4], # 三重東近江線 [Mie Higashi-Omi Line]
"JP-CB->JP-HR": [
5,
11,
], # 南福光連系所・南福光変電所の連系設備 [Minami-Fukumitsu Interconnection / Minami-Fukumitsu Substation Interconnection equipment] and 北陸フェンス [Hokuriku fence]
"JP-HR->JP-KN": [6], # 越前嶺南線 [Echizen Rinan Line]
"JP-CG->JP-KN": [
7
], # 西播東岡山線・山崎智頭線 [Nishiban Higashi Okayama Line / Yamazaki Chitou Line]
"JP-KN->JP-SK": [8], # 阿南紀北直流幹線 [Anan Kihoku DC Trunk Line]
"JP-CG->JP-SK": [9], # 本四連系線 [This Quadruple Interconnection Line]
"JP-CG->JP-KY": [10], # 関門連系線 [Kanmon Interconnection Line]
}
# correct flow direction, if needed
FLOWS_TO_REVERT = ["JP-CB->JP-TK", "JP-CG->JP-KN", "JP-CG->JP-SK"]
SOURCE_URL = "occtonet.occto.or.jp"
EXCHANGE_COLUMNS = ["sortedZoneKeys", "netFlow", "source"]
def _fetch_exchange(
session: Session, datetime: datetime, sorted_zone_keys: str
) -> List[dict]:
exch_id = EXCHANGE_MAPPING[sorted_zone_keys]
# This authorises subsequent calls
cookies = get_cookies(session)
df = pd.DataFrame()
for i in range(len(exch_id)):
form_data = get_form_data(session, exch_id[i], datetime)
_df = get_exchange(session, form_data)
if df.empty:
df = _df
else:
df += _df
df.reset_index()
if sorted_zone_keys in FLOWS_TO_REVERT:
df["netFlow"] = -1 * df["netFlow"]
df["source"] = SOURCE_URL
df["sortedZoneKeys"] = sorted_zone_keys
df = df[EXCHANGE_COLUMNS]
df = df.reset_index()
results = df.to_dict("records")
# For some reason, to_dict converts datetimes to Timestamps
# See https://stackoverflow.com/questions/64171427/pandas-to-dict-converts-datetime-to-timestamp
for result in results:
result["datetime"] = result["datetime"].to_pydatetime()
return results
def fetch_exchange(
zone_key1: str = "JP-TH",
zone_key2: str = "JP-TK",
session: Optional[Session] = None,
target_datetime: Optional[datetime] = None,
logger: Logger = getLogger(__name__),
) -> List[dict]:
"""Requests the last known power exchange (in MW) between two zones."""
if not session:
session = Session()
query_datetime = arrow.get(target_datetime).to("Asia/Tokyo").strftime("%Y/%m/%d")
sorted_zone_keys = "->".join(sorted([zone_key1, zone_key2]))
return _fetch_exchange(session, query_datetime, sorted_zone_keys)
def fetch_exchange_forecast(
zone_key1: str = "JP-TH",
zone_key2: str = "JP-TK",
session: Optional[Session] = None,
target_datetime: Optional[datetime] = None,
logger: Logger = getLogger(__name__),
) -> List[dict]:
"""Gets exchange forecast between two specified zones."""
if not session:
session = Session()
query_datetime = arrow.get(target_datetime).to("Asia/Tokyo").strftime("%Y/%m/%d")
if query_datetime > arrow.get().to("Asia/Tokyo").strftime("%Y/%m/%d"):
raise NotImplementedError(
"Future dates(local time) not implemented for selected exchange"
)
sorted_zone_keys = "->".join(sorted([zone_key1, zone_key2]))
return _fetch_exchange(session, query_datetime, sorted_zone_keys)
def get_cookies(session: Optional[Session] = None) -> cookies.RequestsCookieJar:
if not session:
session = Session()
session.get("http://occtonet.occto.or.jp/public/dfw/RP11/OCCTO/SD/LOGIN_login")
return session.cookies
def get_form_data(session: Session, exchange_id: int, datetime: str) -> Dict[str, str]:
form_data = {
"ajaxToken": "",
"downloadKey": "",
"fwExtention.actionSubType": "headerInput",
"fwExtention.actionType": "reference",
"fwExtention.formId": "CA01S070P",
"fwExtention.jsonString": "",
"fwExtention.pagingTargetTable": "",
"fwExtention.pathInfo": "CA01S070C",
"fwExtention.prgbrh": "0",
"msgArea": "・マージンには需給調整市場の連系線確保量が含まれております。",
"requestToken": "",
"requestTokenBk": "",
"searchReqHdn": "",
"simFlgHdn": "",
"sntkTgtRklCdHdn": "",
"spcDay": datetime,
"spcDayHdn": "",
"tgtRkl": "{:02d}".format(exchange_id),
"tgtRklHdn": "01,北海道・本州間電力連系設備,02,相馬双葉幹線,03,周波数変換設備,04,三重東近江線,05,南福光連系所・南福光変電所の連系設備,06,越前嶺南線,07,西播東岡山線・山崎智頭線,08,阿南紀北直流幹線,09,本四連系線,10,関門連系線,11,北陸フェンス",
"transitionContextKey": "DEFAULT",
"updDaytime": "",
}
r = session.post(
"https://occtonet3.occto.or.jp/public/dfw/RP11/OCCTO/SD/CA01S070C",
data=form_data,
)
response_content = r.json()
if response_content["root"]["errMessage"]:
raise ParserException(
"occtonet.py",
"Headers not available due to {}".format(
response_content["root"]["errMessage"]
),
)
else:
form_data["msgArea"] = response_content["root"]["bizRoot"]["header"]["msgArea"][
"value"
]
form_data["searchReqHdn"] = response_content["root"]["bizRoot"]["header"][
"searchReqHdn"
]["value"]
form_data["spcDayHdn"] = response_content["root"]["bizRoot"]["header"][
"spcDayHdn"
]["value"]
form_data["updDaytime"] = response_content["root"]["bizRoot"]["header"][
"updDaytime"
]["value"]
form_data["fwExtention.actionSubType"] = "ok"
r = session.post(
"https://occtonet3.occto.or.jp/public/dfw/RP11/OCCTO/SD/CA01S070C",
data=form_data,
)
response_content = r.json()
if response_content["root"]["errFields"]:
raise ParserException(
"occtonet.py",
"Request token not available due to {}".format(
response_content["root"]["errFields"]
),
)
else:
form_data["downloadKey"] = response_content["root"]["bizRoot"]["header"][
"downloadKey"
]["value"]
form_data["requestToken"] = response_content["root"]["bizRoot"]["header"][
"requestToken"
]["value"]
return form_data
def _get_exchange(session: Session, form_data: Dict[str, str], columns: List[str]):
def parse_dt(str_dt: str) -> datetime:
return arrow.get(str_dt).replace(tzinfo="Asia/Tokyo").datetime
form_data["fwExtention.actionSubType"] = "download"
r = session.post(
"https://occtonet3.occto.or.jp/public/dfw/RP11/OCCTO/SD/CA01S070C",
data=form_data,
)
r.encoding = "shift-jis"
df = pd.read_csv(StringIO(r.text), delimiter=",")
df = df[columns]
df.columns = ["date", "time", "netFlow"]
df.loc[:, "datetime"] = (df.date + " " + df.time).apply(lambda x: parse_dt(x))
df = df.set_index("datetime")
df = df.drop(columns=["date", "time"])
df = df.dropna()
return df
def get_exchange(session: Session, form_data):
return _get_exchange(session, form_data, ["対象日付", "対象時刻", "潮流実績"])
def get_exchange_fcst(session: Session, form_data):
return _get_exchange(session, form_data, ["対象日付", "対象時刻", "計画潮流(順方向)"])
if __name__ == "__main__":
"""Main method, never used by the Electricity Map backend, but handy for testing."""
print("fetch_exchange(JP-CB, JP-HR) ->")
print(fetch_exchange("JP-CB", "JP-HR")[-3:])
print("fetch_exchange(JP-CG, JP-KY) ->")
print(fetch_exchange("JP-CG", "JP-KY")[-3:])
print("fetch_exchange_forecast(JP-CB, JP-HR) ->")
print(fetch_exchange_forecast("JP-CB", "JP-HR")[-3:])
print("fetch_exchange_forecast(JP-CG, JP-KY) ->")
print(fetch_exchange_forecast("JP-CG", "JP-KY")[-3:])