Skip to content

Commit

Permalink
Small report cli fixes (#691)
Browse files Browse the repository at this point in the history 
* style: capitalize CSV

Signed-off-by: F.N. Claessen <felix@seita.nl>

* fix: reporter is no longer a required parameter

Signed-off-by: F.N. Claessen <felix@seita.nl>

* style: required parameters on top

Signed-off-by: F.N. Claessen <felix@seita.nl>

* style: fix parameter name in docstring

Signed-off-by: F.N. Claessen <felix@seita.nl>

* fix: avoid converting dt type

Signed-off-by: F.N. Claessen <felix@seita.nl>

* feature: add test case with timezone

Signed-off-by: F.N. Claessen <felix@seita.nl>

* feature: add test case for Pandas structure

Signed-off-by: F.N. Claessen <felix@seita.nl>

* fix: type annotations

Signed-off-by: F.N. Claessen <felix@seita.nl>

* fix: use parameter 'end'

Signed-off-by: F.N. Claessen <felix@seita.nl>

* style: fix type annotation

Signed-off-by: F.N. Claessen <felix@seita.nl>

* style: justified indentation

Signed-off-by: F.N. Claessen <felix@seita.nl>

---------

Signed-off-by: F.N. Claessen <felix@seita.nl>
  • Loading branch information
@Flix6x
Flix6x committed May 19, 2023
1 parent b0813c4 commit 1d5d7be
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 50 deletions.
33 changes: 16 additions & 17 deletions flexmeasures/cli/data_add.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -466,21 +466,21 @@ def add_source(name: str, model: str, version: str, source_type: str):
required=True,
type=str,
default=",",
help="[For csv files] Character to delimit columns per row, defaults to comma",
help="[For CSV files] Character to delimit columns per row, defaults to comma",
)
@click.option(
"--decimal",
required=False,
default=".",
type=str,
help="[For csv files] decimal character, e.g. '.' for 10.5",
help="[For CSV files] decimal character, e.g. '.' for 10.5",
)
@click.option(
"--thousands",
required=False,
default=None,
type=str,
help="[For csv files] thousands separator, e.g. '.' for 10.035,2",
help="[For CSV files] thousands separator, e.g. '.' for 10.035,2",
)
@click.option(
"--sheet_number",
Expand Down Expand Up @@ -513,9 +513,9 @@ def add_beliefs(
sheet_number: int | None = None,
**kwargs, # in-code calls to this CLI command can set additional kwargs for use in pandas.read_csv or pandas.read_excel
):
"""Add sensor data from a csv file (also accepts xls or xlsx).
"""Add sensor data from a CSV or Excel file.
To use default settings, structure your csv file as follows:
To use default settings, structure your CSV file as follows:
- One header line (will be ignored!)
- UTC datetimes in 1st column
Expand Down Expand Up @@ -1127,15 +1127,6 @@ def add_schedule_for_storage(

@fm_add_data.command("report")
@with_appcontext
@click.option(
"--reporter",
"reporter_class",
default="PandasReporter",
required=True,
type=click.STRING,
help="Reporter class registered in flexmeasures.data.models.reporting or in an available flexmeasures plugin."
" Use the command `flexmeasures show reporters` to list all the available reporters.",
)
@click.option(
"--sensor-id",
"sensor",
Expand All @@ -1151,6 +1142,14 @@ def add_schedule_for_storage(
type=click.File("r"),
help="Path to the JSON file with the reporter configuration.",
)
@click.option(
"--reporter",
"reporter_class",
default="PandasReporter",
type=click.STRING,
help="Reporter class registered in flexmeasures.data.models.reporting or in an available flexmeasures plugin."
" Use the command `flexmeasures show reporters` to list all the available reporters.",
)
@click.option(
"--start",
"start",
Expand Down Expand Up @@ -1257,7 +1256,7 @@ def add_report( # noqa: C901
):
"""
Create a new report using the Reporter class and save the results
to the database or export them as csv or excel file.
to the database or export them as CSV or Excel file.
"""

# parse timezone into a BaseTzInfo object
Expand Down Expand Up @@ -1291,7 +1290,7 @@ def add_report( # noqa: C901
if end_offset is not None:
if end is None:
end = now
end = apply_offset_chain(now, end_offset)
end = apply_offset_chain(end, end_offset)

# the case of not getting --start, --start-offset or any --last-X flag
if start is None:
Expand Down Expand Up @@ -1327,7 +1326,7 @@ def add_report( # noqa: C901
)
end = now

click.echo(f"Report scope:\n\tstart: {start}\n\tend: {end}")
click.echo(f"Report scope:\n\tstart: {start}\n\tend: {end}")

click.echo(
f"Looking for the Reporter {reporter_class} among all the registered reporters...",
Expand Down
12 changes: 7 additions & 5 deletions flexmeasures/cli/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from typing import Tuple, Mapping
from __future__ import annotations

from typing import Any
from datetime import datetime, timedelta


Expand All @@ -17,9 +19,9 @@ class MsgStyle(object):
"""

SUCCESS: Mapping = {"fg": "green"}
WARN: Mapping = {"fg": "yellow"}
ERROR: Mapping = {"fg": "red"}
SUCCESS: dict[str, Any] = {"fg": "green"}
WARN: dict[str, Any] = {"fg": "yellow"}
ERROR: dict[str, Any] = {"fg": "red"}


class DeprecatedDefaultGroup(DefaultGroup):
Expand Down Expand Up @@ -75,7 +77,7 @@ def get_timerange_from_flag(
last_month: bool = False,
last_year: bool = False,
timezone: pytz.BaseTzInfo = get_timezone(),
) -> Tuple[datetime, datetime]:
) -> tuple[datetime, datetime]:
"""This function returns a time range [start,end] of the last-X period.
See input parameters for more details.
Expand Down
34 changes: 19 additions & 15 deletions flexmeasures/utils/tests/test_time_utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from __future__ import annotations

from datetime import datetime, timedelta

from isodate import duration_isoformat as original_duration_isoformat
Expand Down Expand Up @@ -178,27 +180,29 @@ def test_recent_clocktime_window_invalid_window():
@pytest.mark.parametrize(
"input_date, offset_chain, output_date",
[
(datetime(2023, 5, 17, 10, 15), ",,, , , ", datetime(2023, 5, 17, 10, 15)),
(datetime(2023, 5, 17, 10, 15), "", datetime(2023, 5, 17, 10, 15)),
(datetime(2023, 5, 17, 10, 15), ",,hb, , , ", datetime(2023, 5, 17, 10)),
(datetime(2023, 5, 17, 10, 15), "DB", datetime(2023, 5, 17)),
(datetime(2023, 5, 17, 10, 15), "2D,DB", datetime(2023, 5, 19)),
(datetime(2023, 5, 17, 10, 15), "-2D,DB", datetime(2023, 5, 15)),
# The day of this moment started on May 17th in Amsterdam (but started on May 16th in UTC).
(
datetime(2023, 5, 17, 10, 15),
",,, , , ",
pd.Timestamp(datetime(2023, 5, 17, 10, 15)),
),
(
datetime(2023, 5, 17, 10, 15),
"",
pd.Timestamp(datetime(2023, 5, 17, 10, 15)),
pytz.timezone("Europe/Amsterdam").localize(datetime(2023, 5, 17, 0, 15)),
"DB",
pytz.timezone("Europe/Amsterdam").localize(datetime(2023, 5, 17)),
),
# Check Pandas structure, too.
(
datetime(2023, 5, 17, 10, 15),
",,hb, , , ",
pd.Timestamp(datetime(2023, 5, 17, 10)),
pd.Timestamp("2023-05-17T00:15", tz="Europe/Amsterdam"),
"DB",
pd.Timestamp("2023-05-17", tz="Europe/Amsterdam"),
),
(datetime(2023, 5, 17, 10, 15), "DB", pd.Timestamp(datetime(2023, 5, 17))),
(datetime(2023, 5, 17, 10, 15), "2D,DB", pd.Timestamp(datetime(2023, 5, 19))),
(datetime(2023, 5, 17, 10, 15), "-2D,DB", pd.Timestamp(datetime(2023, 5, 15))),
],
)
def test_apply_offset_chain(
input_date: datetime, offset_chain: str, output_date: pd.Timestamp
input_date: pd.Timestamp | datetime,
offset_chain: str,
output_date: pd.Timestamp | datetime,
):
assert apply_offset_chain(input_date, offset_chain) == output_date
36 changes: 23 additions & 13 deletions flexmeasures/utils/time_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,34 +367,44 @@ def get_max_planning_horizon(resolution: timedelta) -> timedelta | None:


def apply_offset_chain(
time: pd.Timestamp | datetime, offset_chain: str
) -> pd.Timestamp:
dt: pd.Timestamp | datetime, offset_chain: str
) -> pd.Timestamp | datetime:
"""Apply an offset chain to a date.
An offset chain consist of multiple (pandas) offset strings separated by commas. Moreover,
this function implements the offset string "DB", which stands for Day Begin, to
get a date from a datetime, i.e removing time details finer than a day.
get a date from a datetime, i.e. removing time details finer than a day.
Args:
time (pd.Timestamp | datetime)
offsets (str)
dt (pd.Timestamp | datetime)
offset_chain (str)
Returns:
pd.Timestamp: time +
pd.Timestamp | datetime (same type as given dt)
"""
if isinstance(time, datetime):
time = pd.Timestamp(time)

if len(offset_chain) == 0:
return time
return dt

# We need a Pandas structure to apply the offsets
if isinstance(dt, datetime):
_dt = pd.Timestamp(dt)
elif isinstance(dt, pd.Timestamp):
_dt = dt
else:
raise TypeError()

# Apply the offsets
for offset in offset_chain.split(","):
try:
time += to_offset(offset.strip())
_dt += to_offset(offset.strip())
except ValueError:
if offset.strip().lower() == "db": # db = day begin
time = time.floor("D")
_dt = _dt.floor("D")
elif offset.strip().lower() == "hb": # hb = hour begin
time = time.floor("H")
_dt = _dt.floor("H")

return time
# Return output in the same type as the input
if isinstance(dt, datetime):
return _dt.to_pydatetime()
return _dt

0 comments on commit 1d5d7be

Please sign in to comment.