Skip to content

Commit

Permalink
Improve robustness of storage maintenance (#8411)
Browse files Browse the repository at this point in the history 
* Improve robustness of storage maintenance

* Fix tests

* Fix test
  • Loading branch information
@NickM-27
NickM-27 committed Nov 1, 2023
1 parent 89366d7 commit cc79cbc
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 20 deletions.
22 changes: 16 additions & 6 deletions frigate/storage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,13 @@ def reduce_storage_consumption(self) -> None:

# Delete recordings not retained indefinitely
if not keep:
deleted_segments_size += recording.segment_size
Path(recording.path).unlink(missing_ok=True)
deleted_recordings.add(recording.id)
try:
Path(recording.path).unlink(missing_ok=False)
deleted_recordings.add(recording.id)
deleted_segments_size += recording.segment_size
except FileNotFoundError:
# this file was not found so we must assume no space was cleaned up
pass

# check if need to delete retained segments
if deleted_segments_size < hourly_bandwidth:
Expand All @@ -183,9 +187,15 @@ def reduce_storage_consumption(self) -> None:
if deleted_segments_size > hourly_bandwidth:
break

deleted_segments_size += recording.segment_size
Path(recording.path).unlink(missing_ok=True)
deleted_recordings.add(recording.id)
try:
Path(recording.path).unlink(missing_ok=False)
deleted_segments_size += recording.segment_size
deleted_recordings.add(recording.id)
except FileNotFoundError:
# this file was not found so we must assume no space was cleaned up
pass
else:
logger.info(f"Cleaned up {deleted_segments_size} MB of recordings")

logger.debug(f"Expiring {len(deleted_recordings)} recordings")
# delete up to 100,000 at a time
Expand Down
102 changes: 88 additions & 14 deletions frigate/test/test_storage.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import datetime
import logging
import os
import tempfile
import unittest
from unittest.mock import MagicMock

Expand All @@ -26,6 +27,7 @@ def setUp(self):
self.db = SqliteQueueDatabase(TEST_DB)
models = [Event, Recordings]
self.db.bind(models)
self.test_dir = tempfile.mkdtemp()

self.minimal_config = {
"mqtt": {"host": "mqtt"},
Expand Down Expand Up @@ -94,6 +96,7 @@ def test_segment_calculations(self):
rec_bd_id = "1234568.backdoor"
_insert_mock_recording(
rec_fd_id,
os.path.join(self.test_dir, f"{rec_fd_id}.tmp"),
time_keep,
time_keep + 10,
camera="front_door",
Expand All @@ -102,6 +105,7 @@ def test_segment_calculations(self):
)
_insert_mock_recording(
rec_bd_id,
os.path.join(self.test_dir, f"{rec_bd_id}.tmp"),
time_keep + 10,
time_keep + 20,
camera="back_door",
Expand All @@ -123,6 +127,7 @@ def test_segment_calculations_with_zero_segments(self):
rec_fd_id = "1234567.frontdoor"
_insert_mock_recording(
rec_fd_id,
os.path.join(self.test_dir, f"{rec_fd_id}.tmp"),
time_keep,
time_keep + 10,
camera="front_door",
Expand All @@ -141,23 +146,58 @@ def test_storage_cleanup(self):

id = "123456.keep"
time_keep = datetime.datetime.now().timestamp()
_insert_mock_event(id, time_keep, time_keep + 30, True)
_insert_mock_event(
id,
time_keep,
time_keep + 30,
True,
)
rec_k_id = "1234567.keep"
rec_k2_id = "1234568.keep"
rec_k3_id = "1234569.keep"
_insert_mock_recording(rec_k_id, time_keep, time_keep + 10)
_insert_mock_recording(rec_k2_id, time_keep + 10, time_keep + 20)
_insert_mock_recording(rec_k3_id, time_keep + 20, time_keep + 30)
_insert_mock_recording(
rec_k_id,
os.path.join(self.test_dir, f"{rec_k_id}.tmp"),
time_keep,
time_keep + 10,
)
_insert_mock_recording(
rec_k2_id,
os.path.join(self.test_dir, f"{rec_k2_id}.tmp"),
time_keep + 10,
time_keep + 20,
)
_insert_mock_recording(
rec_k3_id,
os.path.join(self.test_dir, f"{rec_k3_id}.tmp"),
time_keep + 20,
time_keep + 30,
)

id2 = "7890.delete"
time_delete = datetime.datetime.now().timestamp() - 360
_insert_mock_event(id2, time_delete, time_delete + 30, False)
rec_d_id = "78901.delete"
rec_d2_id = "78902.delete"
rec_d3_id = "78903.delete"
_insert_mock_recording(rec_d_id, time_delete, time_delete + 10)
_insert_mock_recording(rec_d2_id, time_delete + 10, time_delete + 20)
_insert_mock_recording(rec_d3_id, time_delete + 20, time_delete + 30)
_insert_mock_recording(
rec_d_id,
os.path.join(self.test_dir, f"{rec_d_id}.tmp"),
time_delete,
time_delete + 10,
)
_insert_mock_recording(
rec_d2_id,
os.path.join(self.test_dir, f"{rec_d2_id}.tmp"),
time_delete + 10,
time_delete + 20,
)
_insert_mock_recording(
rec_d3_id,
os.path.join(self.test_dir, f"{rec_d3_id}.tmp"),
time_delete + 20,
time_delete + 30,
)

storage.calculate_camera_bandwidth()
storage.reduce_storage_consumption()
Expand All @@ -176,18 +216,42 @@ def test_storage_cleanup_keeps_retained(self):

id = "123456.keep"
time_keep = datetime.datetime.now().timestamp()
_insert_mock_event(id, time_keep, time_keep + 30, True)
_insert_mock_event(
id,
time_keep,
time_keep + 30,
True,
)
rec_k_id = "1234567.keep"
rec_k2_id = "1234568.keep"
rec_k3_id = "1234569.keep"
_insert_mock_recording(rec_k_id, time_keep, time_keep + 10)
_insert_mock_recording(rec_k2_id, time_keep + 10, time_keep + 20)
_insert_mock_recording(rec_k3_id, time_keep + 20, time_keep + 30)
_insert_mock_recording(
rec_k_id,
os.path.join(self.test_dir, f"{rec_k_id}.tmp"),
time_keep,
time_keep + 10,
)
_insert_mock_recording(
rec_k2_id,
os.path.join(self.test_dir, f"{rec_k2_id}.tmp"),
time_keep + 10,
time_keep + 20,
)
_insert_mock_recording(
rec_k3_id,
os.path.join(self.test_dir, f"{rec_k3_id}.tmp"),
time_keep + 20,
time_keep + 30,
)

time_delete = datetime.datetime.now().timestamp() - 7200
for i in range(0, 59):
id = f"{123456 + i}.delete"
_insert_mock_recording(
f"{123456 + i}.delete", time_delete, time_delete + 600
id,
os.path.join(self.test_dir, f"{id}.tmp"),
time_delete,
time_delete + 600,
)

storage.calculate_camera_bandwidth()
Expand Down Expand Up @@ -219,13 +283,23 @@ def _insert_mock_event(id: str, start: int, end: int, retain: bool) -> Event:


def _insert_mock_recording(
id: str, start: int, end: int, camera="front_door", seg_size=8, seg_dur=10
id: str,
file: str,
start: int,
end: int,
camera="front_door",
seg_size=8,
seg_dur=10,
) -> Event:
"""Inserts a basic recording model with a given id."""
# we must open the file so storage maintainer will delete it
with open(file, "w"):
pass

return Recordings.insert(
id=id,
camera=camera,
path=f"/recordings/{id}",
path=file,
start_time=start,
end_time=end,
duration=seg_dur,
Expand Down

0 comments on commit cc79cbc

Please sign in to comment.