Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor(JP): remove arrow usage #6572

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

refactor(JP): remove arrow usage #6572

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 29 additions & 25 deletions parsers/JP.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
#!/usr/bin/env python3
from datetime import datetime, timedelta
from datetime import datetime, time, timedelta
from logging import Logger, getLogger
from zoneinfo import ZoneInfo

# The arrow library is used to handle datetimes
import arrow
import pandas as pd
from requests import Session

Expand Down Expand Up @@ -36,6 +35,7 @@
"JP-ON": "www.okiden.co.jp/denki/",
}
ZONES_ONLY_LIVE = ["JP-TK", "JP-CB", "JP-SK"]
TIMEZONE = ZoneInfo("Asia/Tokyo")


def get_wind_capacity(datetime: datetime, zone_key, logger: Logger):
Expand Down Expand Up @@ -156,7 +156,12 @@ def fetch_consumption_df(
"""
if target_datetime is not None and zone_key in ZONES_ONLY_LIVE:
raise NotImplementedError("This parser can only fetch live data")
datestamp = arrow.get(target_datetime).to("Asia/Tokyo").strftime("%Y%m%d")

if target_datetime is None:
datestamp = datetime.now(tz=TIMEZONE).strftime("%Y%m%d")
else:
datestamp = target_datetime.replace(tzinfo=TIMEZONE).strftime("%Y%m%d")

consumption_url = {
"JP-HKD": f"http://denkiyoho.hepco.co.jp/area/data/juyo_01_{datestamp}.csv",
"JP-TH": f"https://setsuden.nw.tohoku-epco.co.jp/common/demand/juyo_02_{datestamp}.csv",
Expand Down Expand Up @@ -210,9 +215,14 @@ def fetch_consumption_forecast(
# Currently past dates not implemented for areas with no date in their demand csv files
if target_datetime and zone_key == "JP-HKD":
raise NotImplementedError("Past dates not yet implemented for selected region")
datestamp = arrow.get(target_datetime).to("Asia/Tokyo").strftime("%Y%m%d")

if target_datetime is None:
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When target_datetime == None, arrow.get(target_datetime) returns the current time.

datestamp = datetime.now(tz=TIMEZONE).strftime("%Y%m%d")
else:
datestamp = target_datetime.replace(tzinfo=TIMEZONE).strftime("%Y%m%d")

# Forecasts ahead of current date are not available
if datestamp > arrow.get().to("Asia/Tokyo").strftime("%Y%m%d"):
if datestamp > datetime.now(tz=TIMEZONE).strftime("%Y%m%d"):
raise NotImplementedError(
"Future dates(local time) not implemented for selected region"
)
Expand Down Expand Up @@ -306,9 +316,10 @@ def fetch_price(
df = df[(df["Date"] >= start.date()) & (df["Date"] <= target_datetime.date())]

df["datetime"] = df.apply(
lambda row: arrow.get(row["Date"])
.shift(minutes=30 * (row["Period"] - 1))
.replace(tzinfo="Asia/Tokyo"),
lambda row: (
datetime.combine(row["Date"], time(0, 0))
- timedelta(minutes=30 * (row["Period"] - 1))
).replace(tzinfo=TIMEZONE),
axis=1,
)

Expand All @@ -329,26 +340,19 @@ def fetch_price(
return data


def parse_dt(row):
def parse_dt(row) -> datetime:
"""Parses timestamps from date and time."""
if "AM" in row["Time"] or "PM" in row["Time"]:
timestamp = (
arrow.get(
" ".join([row["Date"], row["Time"]]).replace("/", "-"),
"YYYY-M-D H:mm A",
)
.replace(tzinfo="Asia/Tokyo")
.datetime
date = row["Date"]
time = row["Time"]
Comment on lines +345 to +346
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it not be easier to parse these directly using the date and time libraries and then just combine them instead of using string manipulation?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks only datetime library has strptime() but there is no string parser in date and time standard libraries (ref. https://docs.python.org/3.10/library/datetime.html#strftime-strptime-behavior). I wonder if there's another method other than datetime.strptime().

Copy link
Member

@VIKTORVAV99 VIKTORVAV99 Apr 7, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

datetime.time has their own parsing functions, same with date.

https://docs.python.org/3.10/library/time.html#time.strptime

You could also use the date time library directly on each of them and just call .date() or .time() to get the individual parts.

datetime_str = f"{date} {time}".replace("/", "-")
if "AM" in time or "PM" in time:
return datetime.strptime(datetime_str, "%Y-%m-%d %I:%M %p").replace(
tzinfo=ZoneInfo("Asia/Tokyo")
)
else:
timestamp = (
arrow.get(
" ".join([row["Date"], row["Time"]]).replace("/", "-"), "YYYY-M-D H:mm"
)
.replace(tzinfo="Asia/Tokyo")
.datetime
return datetime.strptime(datetime_str, "%Y-%m-%d %H:%M").replace(
tzinfo=ZoneInfo("Asia/Tokyo")
)
return timestamp


if __name__ == "__main__":
Expand Down