Skip to content

Commit

Permalink
Completely rework the sequencing machinery
Browse files Browse the repository at this point in the history 
Features:

- Allow relative timestamps / time ranges also for ``start`` and
  ``stop`` parameters.

Plumbing:

- Large refactoring. All utility functions related to times and time
  rangers are now located in the `timeutil.py` and `model.py` files.
- Add new model `AnimationFrame`
- Fix frame file sort order: Add `index` to each filename in order to
  reflect the position of the `AnimationSequence` in a list of many.
- Add "--dry-run" parameter
- Add a few more software tests
  • Loading branch information
@amotl
amotl committed Nov 20, 2021
1 parent 04f0e0a commit e4b0b4e
Show file tree
Hide file tree
Showing 18 changed files with 709 additions and 385 deletions.
3 changes: 3 additions & 0 deletions CHANGES.rst
View file
Expand Up @@ -7,6 +7,9 @@ in progress
===========
- Improve scope of values for ``every`` parameter. It will now accept relative
humanized timestamps like ``2m30s``, ``1d12h`` or ``1.5 days``.
- Allow relative timestamps / time ranges also for ``start`` and ``stop``
parameters. Accepted are humanized values like outlined above (``2m30s``),
combined with, e.g., ``stop=start+2m30s`` or ``start=-1h, stop=now``.


2021-11-17 0.6.0
Expand Down
18 changes: 12 additions & 6 deletions doc/backlog.rst
View file
Expand Up @@ -8,14 +8,12 @@ Prio 1.25
*********
- [x] Rename ``AnimationScenario.steps`` to ``AnimationScenario.sequences``
- [x] Rename ``dtstart``/``dtuntil`` to ``start``/``stop`` and ``interval`` to ``every``
- [o] Allow relative time range for ``stop`` parameter
> The every parameter supports all valid duration units, including calendar months (1mo) and years (1y).
- [x] Allow relative time range for ``stop`` parameter
> The ``every`` parameter supports all valid duration units, including calendar months (1mo) and years (1y).
> -- https://docs.influxdata.com/flux/v0.x/spec/types/#duration-types
- [o] Mix relative with absolute timestamps: ``range(start:2019-01-31T23:00:00Z, stop:-1h)``
- [x] Mix relative with absolute timestamps: ``range(start:2019-01-31T23:00:00Z, stop:-1h)``
-- https://community.hiveeyes.org/t/improving-time-range-control-for-grafanimate/1783/11
- [o] README: Drop some words about "Animation Speed" (--framerate)
- [o] Specify number of frames to capture, instead of --stop
- [o] Add ``--reverse`` parameter
- [o] Implement "ad-hoc" mode

Until implemented, please use scenario mode.
Expand All @@ -29,11 +27,19 @@ Prio 1.25
-- https://community.hiveeyes.org/t/improving-time-range-control-for-grafanimate/1783/13

- [o] Panel spinner is visible again
- [o] Avoid collisions in output directory, e.g. take sequencing mode into account
- [o] Avoid collisions in output directory, e.g. take sequencing mode and start/stop timestamps into account
- [o] Final tests
- [o] Release 0.7.0


********
Prio 1.3
********
- [o] Support for months and years: Follow up on https://github.com/onegreyonewhite/pytimeparse2/pull/1
- [o] Specify number of frames to capture, instead of --stop
- [o] Add ``--reverse`` parameter; hm, or use negative ``stop`` parameter instead


********
Prio 1.5
********
Expand Down
183 changes: 23 additions & 160 deletions grafanimate/animations.py
View file
@@ -1,39 +1,24 @@
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
# (c) 2018-2021 Andreas Motl <andreas.motl@panodata.org>
# License: GNU Affero General Public License, Version 3
import logging
import time
from datetime import timedelta
from operator import attrgetter
from typing import Tuple

from dateutil.relativedelta import relativedelta
from dateutil.rrule import (
DAILY,
HOURLY,
MINUTELY,
MONTHLY,
SECONDLY,
WEEKLY,
YEARLY,
rrule,
)
from munch import munchify
from pytimeparse2 import parse as parse_human_time

from munch import Munch, munchify

from grafanimate.grafana import GrafanaWrapper
from grafanimate.model import AnimationSequence, SequencingMode
from grafanimate.util import get_relativedelta
from grafanimate.model import AnimationFrame, AnimationSequence

logger = logging.getLogger(__name__)


class SequentialAnimation:
def __init__(self, grafana: GrafanaWrapper, dashboard_uid=None, options=None):
def __init__(self, grafana: GrafanaWrapper, dashboard_uid: str = None, options: Munch = None):

self.grafana = grafana
self.dashboard_uid = dashboard_uid
self.options = options or None
self.dry_run: bool = self.options.get("dry-run", False)

def start(self):
self.log("Opening dashboard")
Expand All @@ -46,43 +31,18 @@ def log(self, message):

def run(self, sequence: AnimationSequence):

self.log("Starting animation: {}".format(sequence))

# Destructure `AnimationSequence` instance.
start, stop, every, mode = attrgetter("start", "stop", "every", "mode")(sequence)

if start > stop:
message = "Timestamp start={} is after stop={}".format(start, stop)
raise ValueError(message)

rr_freq, rr_interval, dtdelta = self.get_freq_delta(every)

# until = datetime.now()
if mode == SequencingMode.CUMULATIVE:
stop += dtdelta

# Compute complete date range.
logger.info("Creating rrule: dtstart=%s, until=%s, freq=%s, interval=%s", start, stop, rr_freq, rr_interval)
daterange = list(rrule(dtstart=start, until=stop, freq=rr_freq, interval=rr_interval))
# logger.info('Date range is: %s', daterange)
if not isinstance(sequence, AnimationSequence):
return

# Iterate date range.
for date in daterange:

logger.info("=" * 42)
logger.info("Datetime step: %s", date)

# Compute start and end dates based on mode.
self.log("Starting animation: {}".format(sequence))

if mode == SequencingMode.WINDOW:
start = date
stop = date + dtdelta
frame: AnimationFrame = None
for frame in sequence.get_frames():

elif mode == SequencingMode.CUMULATIVE:
stop = date
# logger.info("=" * 42)

# Render image.
image = self.render(start, stop, every)
image = self.render(frame)

# Build item model.
item = munchify(
Expand All @@ -91,130 +51,33 @@ def run(self, sequence: AnimationSequence):
"grafana": self.grafana,
"scenario": self.options["scenario"],
"dashboard": self.dashboard_uid,
"every": every,
"every": frame.timerange.recurrence.every,
},
"data": {
"start": start,
"stop": stop,
"start": frame.timerange.start,
"stop": frame.timerange.stop,
"image": image,
},
"frame": frame,
}
)

yield item

if self.options["exposure-time"] > 0:
logger.info("Waiting for {} seconds (exposure time)".format(self.options["exposure-time"]))
time.sleep(self.options["exposure-time"])

yield item

self.log("Animation finished")

@staticmethod
def get_freq_delta(interval: str) -> Tuple[int, int, timedelta]:

rr_freq = MINUTELY
rr_interval = 1

# 1. Attempt to parse time using `pytimeparse` module.
# https://pypi.org/project/pytimeparse/
duration = parse_human_time(interval)
if duration:
delta = get_relativedelta(seconds=duration)

if delta.years:
rr_freq = YEARLY
rr_interval = delta.years
elif delta.months:
rr_freq = MONTHLY
rr_interval = delta.months
elif delta.days:
rr_freq = DAILY
rr_interval = delta.days
elif delta.hours:
rr_freq = HOURLY
rr_interval = delta.hours
elif delta.minutes:
rr_freq = MINUTELY
rr_interval = delta.minutes
else:
rr_freq = SECONDLY
rr_interval = delta.seconds

if rr_freq != SECONDLY:
delta -= relativedelta(seconds=1)

return rr_freq, rr_interval, delta

# 2. Compute parameters from specific labels, expression periods.

# Secondly
if interval == "secondly":
rr_freq = SECONDLY
delta = timedelta(seconds=1)

# Minutely
elif interval == "minutely":
rr_freq = MINUTELY
delta = timedelta(minutes=1) - timedelta(seconds=1)

# Each 5 minutes
elif interval == "5min":
rr_freq = MINUTELY
rr_interval = 5
delta = timedelta(minutes=5) - timedelta(seconds=1)

# Each 10 minutes
elif interval == "10min":
rr_freq = MINUTELY
rr_interval = 10
delta = timedelta(minutes=10) - timedelta(seconds=1)

# Each 30 minutes
elif interval == "30min":
rr_freq = MINUTELY
rr_interval = 30
delta = timedelta(minutes=30) - timedelta(seconds=1)

# Hourly
elif interval == "hourly":
rr_freq = HOURLY
delta = timedelta(hours=1) - timedelta(seconds=1)

# Daily
elif interval == "daily":
rr_freq = DAILY
delta = timedelta(days=1) - timedelta(seconds=1)

# Weekly
elif interval == "weekly":
rr_freq = WEEKLY
delta = timedelta(weeks=1) - timedelta(seconds=1)

# Monthly
elif interval == "monthly":
rr_freq = MONTHLY
delta = relativedelta(months=+1) - relativedelta(seconds=1)

# Yearly
elif interval == "yearly":
rr_freq = YEARLY
delta = relativedelta(years=+1) - relativedelta(seconds=1)

else:
raise ValueError('Unknown interval "{}"'.format(interval))

if isinstance(delta, timedelta):
delta = get_relativedelta(seconds=delta.total_seconds())

return rr_freq, rr_interval, delta

def render(self, start, stop, every):
def render(self, frame: AnimationFrame):

logger.debug("Adjusting time range control")
self.grafana.timewarp(start, stop, every)
self.grafana.timewarp(frame, self.dry_run)

logger.debug("Rendering image")
return self.make_image()
if not self.dry_run:
return self.make_image()

def make_image(self):
image = self.grafana.render_image()
Expand Down
9 changes: 5 additions & 4 deletions grafanimate/commands.py
View file
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# (c) 2018 Andreas Motl <andreas@hiveeyes.org>
# (c) 2018-2021 Andreas Motl <andreas.motl@panodata.org>
# License: GNU Affero General Public License, Version 3
import json
import logging
Expand Down Expand Up @@ -91,7 +91,7 @@ def run():
FFmpeg's `-filter_complex` options. [default: 10]
--gif-width=<pixel> Width of the gif in pixels. [default: 480]
--dry-run Enable dry-run mode
--debug Enable debug logging
-h --help Show this screen
Expand Down Expand Up @@ -179,5 +179,6 @@ def run():

# Run rendering sequences, produce composite media artifacts.
scenario.dashboard_title = grafana.get_dashboard_title()
results = produce_artifacts(input=storage.workdir, output=output, scenario=scenario, options=render_options)
log.info("Produced %s results\n%s", len(results), json.dumps(results, indent=2))
if not options.dry_run:
results = produce_artifacts(input=storage.workdir, output=output, scenario=scenario, options=render_options)
log.info("Produced %s results\n%s", len(results), json.dumps(results, indent=2))
12 changes: 7 additions & 5 deletions grafanimate/core.py
View file
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# (c) 2018-2021 Andreas Motl <andreas@hiveeyes.org>
# (c) 2018-2021 Andreas Motl <andreas.motl@panodata.org>
# License: GNU Affero General Public License, Version 3
import logging
from pathlib import Path
Expand Down Expand Up @@ -82,7 +82,7 @@ def resolve_reference(module, symbol):

def run_animation_scenario(scenario: AnimationScenario, grafana: GrafanaWrapper, options: Munch) -> TemporaryStorage:

log.info("Running animation scenario {}".format(scenario))
log.info(f"Running animation scenario at {scenario.grafana_url}, with dashboard UID {scenario.dashboard_uid}")

storage = TemporaryStorage()

Expand All @@ -105,9 +105,11 @@ def run_animation_scenario(scenario: AnimationScenario, grafana: GrafanaWrapper,
animation.start()

# Run animation scenario.
for sequence in scenario.sequences:
results = animation.run(sequence)
storage.save_items(results)
for index, sequence in enumerate(scenario.sequences):
sequence.index = index
results = list(animation.run(sequence))
if not options.dry_run:
storage.save_items(results)

return storage

Expand Down

0 comments on commit e4b0b4e

Please sign in to comment.