Metadata Timeline (#6194)

* Create timeline table

* Fix indexes

* Add other fields

* Adjust schema to be less descriptive

* Handle timeline queue from tracked object data

* Setup timeline queue in events

* Add source id for index

* Add other fields

* Fixes

* Formatting

* Store better data

* Add api with filtering

* Setup basic UI for timeline in events

* Cleanups

* Add recordings snapshot url

* Start working on timeline ui

* Add tooltip with info

* Improve icons

* Fix start time with clip

* Move player logic back to clips

* Make box in timeline relative coordinates

* Make region relative

* Get box overlay working

* Remove overlay when playing again

* Add disclaimer when selecting overlay points

* Add docs for new apis

* Fix mobile

* Fix docs

* Change color of bottom center box

* Fix vscode formatting

docs/docs/integrations/api.md

@@ -168,6 +168,16 @@ Events from the database. Accepts the following query string parameters:
 | `include_thumbnails` | int  | Include thumbnails in the response (0 or 1)   |
 | `in_progress`        | int  | Limit to events in progress (0 or 1)          |
 
+### `GET /api/timeline`
+
+Timeline of key moments of an event(s) from the database. Accepts the following query string parameters:
+
+| param                | Type | Description                                   |
+| -------------------- | ---- | --------------------------------------------- |
+| `camera`             | int  | Name of camera                                |
+| `source_id`          | str  | ID of tracked object                          |
+| `limit`              | int  | Limit the number of events returned           |
+
 ### `GET /api/events/summary`
 
 Returns summary data for events in the database. Used by the Home Assistant integration.
@@ -233,6 +243,10 @@ Accepts the following query string parameters, but they are only applied when an
 
 Returns the snapshot image from the latest event for the given camera and label combo. Using `any` as the label will return the latest thumbnail regardless of type.
 
+### `GET /api/<camera_name>/recording/<frame_time>/snapshot.png`
+
+Returns the snapshot image from the specific point in that cameras recordings.
+
 ### `GET /clips/<camera>-<id>.jpg`
 
 JPG snapshot for the given camera and event id.

frigate/app.py

@@ -23,13 +23,14 @@
 from frigate.events import EventCleanup, EventProcessor
 from frigate.http import create_app
 from frigate.log import log_process, root_configurer
-from frigate.models import Event, Recordings
+from frigate.models import Event, Recordings, Timeline
 from frigate.object_processing import TrackedObjectProcessor
 from frigate.output import output_frames
 from frigate.plus import PlusApi
 from frigate.record import RecordingCleanup, RecordingMaintainer
 from frigate.stats import StatsEmitter, stats_init
 from frigate.storage import StorageMaintainer
+from frigate.timeline import TimelineProcessor
 from frigate.version import VERSION
 from frigate.video import capture_camera, track_camera
 from frigate.watchdog import FrigateWatchdog
@@ -135,6 +136,9 @@ def init_queues(self) -> None:
         # Queue for recordings info
         self.recordings_info_queue: Queue = mp.Queue()
 
+        # Queue for timeline events
+        self.timeline_queue: Queue = mp.Queue()
+
     def init_database(self) -> None:
         # Migrate DB location
         old_db_path = os.path.join(CLIPS_DIR, "frigate.db")
@@ -154,7 +158,7 @@ def init_database(self) -> None:
         migrate_db.close()
 
         self.db = SqliteQueueDatabase(self.config.database.path)
-        models = [Event, Recordings]
+        models = [Event, Recordings, Timeline]
         self.db.bind(models)
 
     def init_stats(self) -> None:
@@ -286,12 +290,19 @@ def start_camera_capture_processes(self) -> None:
             capture_process.start()
             logger.info(f"Capture process started for {name}: {capture_process.pid}")
 
+    def start_timeline_processor(self) -> None:
+        self.timeline_processor = TimelineProcessor(
+            self.config, self.timeline_queue, self.stop_event
+        )
+        self.timeline_processor.start()
+
     def start_event_processor(self) -> None:
         self.event_processor = EventProcessor(
             self.config,
             self.camera_metrics,
             self.event_queue,
             self.event_processed_queue,
+            self.timeline_queue,
             self.stop_event,
         )
         self.event_processor.start()
@@ -384,6 +395,7 @@ def start(self) -> None:
         self.start_storage_maintainer()
         self.init_stats()
         self.init_web_server()
+        self.start_timeline_processor()
         self.start_event_processor()
         self.start_event_cleanup()
         self.start_recording_maintainer()

frigate/events.py

@@ -3,14 +3,14 @@
 import os
 import queue
 import threading
-import time
 from pathlib import Path
 
 from peewee import fn
 
-from frigate.config import EventsConfig, FrigateConfig, RecordConfig
+from frigate.config import EventsConfig, FrigateConfig
 from frigate.const import CLIPS_DIR
 from frigate.models import Event
+from frigate.timeline import TimelineSourceEnum
 from frigate.types import CameraMetricsTypes
 
 from multiprocessing.queues import Queue
@@ -48,6 +48,7 @@ def __init__(
         camera_processes: dict[str, CameraMetricsTypes],
         event_queue: Queue,
         event_processed_queue: Queue,
+        timeline_queue: Queue,
         stop_event: MpEvent,
     ):
         threading.Thread.__init__(self)
@@ -56,6 +57,7 @@ def __init__(
         self.camera_processes = camera_processes
         self.event_queue = event_queue
         self.event_processed_queue = event_processed_queue
+        self.timeline_queue = timeline_queue
         self.events_in_process: Dict[str, Event] = {}
         self.stop_event = stop_event
 
@@ -73,6 +75,16 @@ def run(self) -> None:
 
             logger.debug(f"Event received: {event_type} {camera} {event_data['id']}")
 
+            self.timeline_queue.put(
+                (
+                    camera,
+                    TimelineSourceEnum.tracked_object,
+                    event_type,
+                    self.events_in_process.get(event_data["id"]),
+                    event_data,
+                )
+            )
+
             event_config: EventsConfig = self.config.cameras[camera].record.events
 
             if event_type == "start":

frigate/http.py

@@ -33,7 +33,7 @@
 
 from frigate.config import FrigateConfig
 from frigate.const import CLIPS_DIR, MAX_SEGMENT_DURATION, RECORD_DIR
-from frigate.models import Event, Recordings
+from frigate.models import Event, Recordings, Timeline
 from frigate.object_processing import TrackedObject
 from frigate.stats import stats_snapshot
 from frigate.util import (
@@ -414,6 +414,42 @@ def event_thumbnail(id, max_cache_age=2592000):
     return response
 
 
+@bp.route("/timeline")
+def timeline():
+    camera = request.args.get("camera", "all")
+    source_id = request.args.get("source_id", type=str)
+    limit = request.args.get("limit", 100)
+
+    clauses = []
+
+    selected_columns = [
+        Timeline.timestamp,
+        Timeline.camera,
+        Timeline.source,
+        Timeline.source_id,
+        Timeline.class_type,
+        Timeline.data,
+    ]
+
+    if camera != "all":
+        clauses.append((Timeline.camera == camera))
+
+    if source_id:
+        clauses.append((Timeline.source_id == source_id))
+
+    if len(clauses) == 0:
+        clauses.append((True))
+
+    timeline = (
+        Timeline.select(*selected_columns)
+        .where(reduce(operator.and_, clauses))
+        .order_by(Timeline.timestamp.asc())
+        .limit(limit)
+    )
+
+    return jsonify([model_to_dict(t) for t in timeline])
+
+
 @bp.route("/<camera_name>/<label>/best.jpg")
 @bp.route("/<camera_name>/<label>/thumbnail.jpg")
 def label_thumbnail(camera_name, label):
@@ -924,6 +960,53 @@ def latest_frame(camera_name):
         return "Camera named {} not found".format(camera_name), 404
 
 
+@bp.route("/<camera_name>/recordings/<frame_time>/snapshot.png")
+def get_snapshot_from_recording(camera_name: str, frame_time: str):
+    if camera_name not in current_app.frigate_config.cameras:
+        return "Camera named {} not found".format(camera_name), 404
+
+    frame_time = float(frame_time)
+    recording_query = (
+        Recordings.select()
+        .where(
+            ((frame_time > Recordings.start_time) & (frame_time < Recordings.end_time))
+        )
+        .where(Recordings.camera == camera_name)
+    )
+
+    try:
+        recording: Recordings = recording_query.get()
+        time_in_segment = frame_time - recording.start_time
+
+        ffmpeg_cmd = [
+            "ffmpeg",
+            "-hide_banner",
+            "-loglevel",
+            "warning",
+            "-ss",
+            f"00:00:{time_in_segment}",
+            "-i",
+            recording.path,
+            "-frames:v",
+            "1",
+            "-c:v",
+            "png",
+            "-f",
+            "image2pipe",
+            "-",
+        ]
+
+        process = sp.run(
+            ffmpeg_cmd,
+            capture_output=True,
+        )
+        response = make_response(process.stdout)
+        response.headers["Content-Type"] = "image/png"
+        return response
+    except DoesNotExist:
+        return "Recording not found for {} at {}".format(camera_name, frame_time), 404
+
+
 @bp.route("/recordings/storage", methods=["GET"])
 def get_recordings_storage_usage():
     recording_stats = stats_snapshot(

frigate/models.py

@@ -32,6 +32,15 @@ class Event(Model):  # type: ignore[misc]
     plus_id = CharField(max_length=30)
 
 
+class Timeline(Model):  # type: ignore[misc]
+    timestamp = DateTimeField()
+    camera = CharField(index=True, max_length=20)
+    source = CharField(index=True, max_length=20)  # ex: tracked object, audio, external
+    source_id = CharField(index=True, max_length=30)
+    class_type = CharField(max_length=50)  # ex: entered_zone, audio_heard
+    data = JSONField()  # ex: tracked object id, region, box, etc.
+
+
 class Recordings(Model):  # type: ignore[misc]
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)