Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
DVR: space maintenance cleanups, add high watermark handling
  • Loading branch information
@perexg
perexg committed Dec 3, 2015
1 parent 77f6608 commit 18a4a8c
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 42 deletions.
17 changes: 10 additions & 7 deletions src/dvr/dvr.h
View file
Expand Up @@ -54,7 +54,8 @@ typedef struct dvr_config {
uint32_t dvr_extra_time_post;
uint32_t dvr_update_window;
int dvr_running;
uint32_t dvr_cleanup_threshold;
uint32_t dvr_cleanup_threshold_low;
uint32_t dvr_cleanup_threshold_high;

muxer_config_t dvr_muxcnf;

Expand Down Expand Up @@ -129,13 +130,15 @@ typedef enum {
DVR_RET_1WEEK = 7,
DVR_RET_2WEEK = 14,
DVR_RET_3WEEK = 21,
DVR_RET_1MONTH = 30,
DVR_RET_2MONTH = 60,
DVR_RET_3MONTH = 90,
DVR_RET_6MONTH = 180,
DVR_RET_1YEAR = 365,
DVR_RET_1MONTH = (30+1),
DVR_RET_2MONTH = (60+1),
DVR_RET_3MONTH = (90+2),
DVR_RET_6MONTH = (180+2),
DVR_RET_1YEAR = (365+1),
DVR_RET_2YEARS = (2*365+1),
DVR_RET_3YEARS = (3*366+1),

This comment has been minimized.

Sign in to view

Copy link
@Glenn-1990

Glenn-1990 Dec 3, 2015

Contributor

I wonder why these +1, and +2 and 366?
Is that for correcting the days with multiple months?
If so:
-1 month should be 30
-2 month should be 90+1
-6 month should be 180+3
-1 year should be 365
-3 years should be 3*365+1

This comment has been minimized.

Sign in to view

Copy link
@perexg

perexg Dec 3, 2015

Author Contributor

I take maximal possible numbers (31 days in month and possible leap-year). It's better to have one day plus than minus in my opinion.

This comment has been minimized.

Sign in to view

Copy link
@perexg

perexg Dec 3, 2015

Author Contributor

And right - I overlooked 366 in 3years..

This comment has been minimized.

Sign in to view

Copy link
@linuxstb

linuxstb Dec 3, 2015

Contributor

It seems wrong to tell the user that the retention is, for example "3 months" and then actually implement that as a fixed number of days, which may or may not be equal to 3 months.

As a developer who has just worked on a project involving dates ands intervals, I would suggest to keep it simple, and don't let the user specify numbers of months or years, especially if your internal representation is just days. As a user, I would be just as happy to say "52 weeks" as "1 year", or "4 weeks" instead of 1 month etc
DVR_RET_ONREMOVE = UINT32_MAX-1, // for retention only
DVR_RET_SPACENEED = UINT32_MAX-1, // for removal only
DVR_RET_SPACE = UINT32_MAX-1, // for removal only
DVR_RET_FOREVER = UINT32_MAX
} dvr_retention_t;

Expand Down
29 changes: 21 additions & 8 deletions src/dvr/dvr_config.c
View file
Expand Up @@ -186,7 +186,8 @@ dvr_config_create(const char *name, const char *uuid, htsmsg_t *conf)
cfg->dvr_warm_time = 30;
cfg->dvr_update_window = 24 * 3600;
cfg->dvr_pathname = strdup("$t$n.$x");
cfg->dvr_cleanup_threshold = 2000;
cfg->dvr_cleanup_threshold_low = 200;
cfg->dvr_cleanup_threshold_high = 2000;

/* Muxer config */
cfg->dvr_muxcnf.m_cache = MC_CACHE_DONTKEEP;
Expand Down Expand Up @@ -516,8 +517,10 @@ dvr_config_save(dvr_config_t *cfg)
lock_assert(&global_lock);

dvr_config_storage_check(cfg);
if (cfg->dvr_cleanup_threshold < 100)
cfg->dvr_cleanup_threshold = 100; // as checking is only periodically, lower is not save
if (cfg->dvr_cleanup_threshold_low < 50)
cfg->dvr_cleanup_threshold_low = 50; // as checking is only periodically, lower is not save
if (cfg->dvr_cleanup_threshold_high < cfg->dvr_cleanup_threshold_high)
cfg->dvr_cleanup_threshold_high = cfg->dvr_cleanup_threshold_low + 50;
if (cfg->dvr_removal_days != DVR_RET_FOREVER &&
cfg->dvr_removal_days > cfg->dvr_retention_days)
cfg->dvr_retention_days = DVR_RET_ONREMOVE;
Expand Down Expand Up @@ -714,7 +717,7 @@ dvr_config_class_removal_list ( void *o, const char *lang )
{ N_("3 months"), DVR_RET_3MONTH },
{ N_("6 months"), DVR_RET_6MONTH },
{ N_("1 year"), DVR_RET_1YEAR },
{ N_("Until space needed"), DVR_RET_SPACENEED },
{ N_("Maintained space"), DVR_RET_SPACE },
{ N_("Forever"), DVR_RET_FOREVER },
};
return strtab2htsmsg_u32(tab, 1, lang);
Expand All @@ -735,6 +738,8 @@ dvr_config_class_retention_list ( void *o, const char *lang )
{ N_("3 months"), DVR_RET_3MONTH },
{ N_("6 months"), DVR_RET_6MONTH },
{ N_("1 year"), DVR_RET_1YEAR },
{ N_("2 years"), DVR_RET_2YEARS },
{ N_("3 years"), DVR_RET_3YEARS },
{ N_("On file removal"), DVR_RET_ONREMOVE },
{ N_("Forever"), DVR_RET_FOREVER },
};
Expand Down Expand Up @@ -979,10 +984,18 @@ const idclass_t dvr_config_class = {
},
{
.type = PT_U32,
.id = "storage-cleanup",
.name = N_("\"Until space needed\" cleanup below (MB)"),
.off = offsetof(dvr_config_t, dvr_cleanup_threshold),
.def.i = 2000, //2000 MB
.id = "storage-cleanup-low",
.name = N_("Maintain free storage space (MiB)"),
.off = offsetof(dvr_config_t, dvr_cleanup_threshold_low),
.def.i = 200,
.group = 2,
},
{
.type = PT_U32,
.id = "storage-cleanup-high",
.name = N_("Maintain used storage space (MiB)"),
.off = offsetof(dvr_config_t, dvr_cleanup_threshold_high),
.def.i = 2000,
.group = 2,
},
{
Expand Down
8 changes: 4 additions & 4 deletions src/dvr/dvr_db.c
View file
Expand Up @@ -277,9 +277,9 @@ dvr_entry_get_removal_string ( dvr_entry_t *de )
char buf[24];
uint32_t removal = dvr_entry_get_removal_days(de);

if (removal < DVR_RET_SPACENEED)
if (removal < DVR_RET_SPACE)
snprintf(buf, sizeof(buf), "%i days", removal);
else if (removal == DVR_RET_SPACENEED)
else if (removal == DVR_RET_SPACE)
return strdup("Until space needed");
else
return strdup("Forever");
Expand Down Expand Up @@ -388,7 +388,7 @@ dvr_entry_retention_timer(dvr_entry_t *de)
uint32_t retention = dvr_entry_get_retention_days(de);

stop = de->de_stop + removal * (time_t)86400;
if ((removal > 0 || retention == 0) && removal < DVR_RET_SPACENEED) {
if ((removal > 0 || retention == 0) && removal < DVR_RET_SPACE) {
if (stop > dispatch_clock) {
dvr_entry_retention_arm(de, dvr_timer_remove_files, stop);
return;
Expand Down Expand Up @@ -2219,7 +2219,7 @@ dvr_entry_class_removal_list ( void *o, const char *lang )
{ N_("3 months"), DVR_RET_3MONTH },
{ N_("6 months"), DVR_RET_6MONTH },
{ N_("1 year"), DVR_RET_1YEAR },
{ N_("Until space needed"), DVR_RET_SPACENEED },
{ N_("Maintained space"), DVR_RET_SPACE },
{ N_("Forever"), DVR_RET_FOREVER },
};
return strtab2htsmsg_u32(tab, 1, lang);
Expand Down
62 changes: 39 additions & 23 deletions src/dvr/dvr_vfsmgr.c
View file
Expand Up @@ -38,6 +38,9 @@
#include <sys/statvfs.h>
#endif

#define MIB(v) ((int64_t)v*((int64_t)1024*1024))
#define TOMIB(v) (v/((int64_t)1024*1024))

static int dvr_disk_space_config_idx;
static int dvr_disk_space_config_size;
static time_t dvr_disk_space_config_lastdelete;
Expand All @@ -56,7 +59,7 @@ dvr_disk_space_cleanup(dvr_config_t *cfg)
{
dvr_entry_t *de, *oldest;
time_t stoptime;
int64_t requiredBytes, availBytes;
int64_t requiredBytes, maximalBytes, availBytes, usedBytes, diskBytes;
int64_t clearedBytes = 0, fileSize;
unsigned long int filesystemId;
struct statvfs diskdata;
Expand All @@ -70,7 +73,10 @@ dvr_disk_space_cleanup(dvr_config_t *cfg)

filesystemId = diskdata.f_fsid;
availBytes = diskdata.f_bsize * (int64_t)diskdata.f_bavail;
requiredBytes = (int64_t)cfg->dvr_cleanup_threshold*(int64_t)1024*(int64_t)1024;
requiredBytes = MIB(cfg->dvr_cleanup_threshold_low);
diskBytes = diskdata.f_bsize * (int64_t)diskdata.f_blocks;
usedBytes = diskBytes - availBytes;
maximalBytes = MIB(cfg->dvr_cleanup_threshold_high);
configName = cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile";

/* When deleting a file from the disk, the system needs some time to actually do this */
Expand All @@ -81,16 +87,17 @@ dvr_disk_space_cleanup(dvr_config_t *cfg)
return -1;
}

if (diskdata.f_bsize * (int64_t)diskdata.f_blocks < requiredBytes) {
tvhlog(LOG_WARNING, "dvr","disk space cleanup for config \"%s\", required free space \"%ld MB\" is smaller than the total disk size!",
if (diskBytes < requiredBytes) {
tvhlog(LOG_WARNING, "dvr","disk space cleanup for config \"%s\", required free space \"%ld MiB\" is smaller than the total disk size!",
configName, requiredBytes/(int64_t)1024/(int64_t)1024);
return -1;
if (maximalBytes >= usedBytes)
return -1;
}

tvhlog(LOG_INFO, "dvr","disk space cleanup for config \"%s\", required free space \"%ld MB\", current free space \"%ld MB\"",
configName, requiredBytes/(int64_t)1024/(int64_t)1024, availBytes/(int64_t)1024/(int64_t)1024);
tvhlog(LOG_INFO, "dvr","disk space cleanup for config \"%s\", required/current free space \"%ld/%ld MiB\", required/current used space \"%ld/%ld MB\"",
configName, TOMIB(requiredBytes), TOMIB(availBytes), TOMIB(maximalBytes), TOMIB(usedBytes));

while (availBytes < requiredBytes) {
while (availBytes < requiredBytes || maximalBytes < usedBytes) {
oldest = NULL;
stoptime = dispatch_clock;

Expand All @@ -102,7 +109,7 @@ dvr_disk_space_cleanup(dvr_config_t *cfg)
if (dvr_entry_get_stop_time(de) > stoptime)
continue;

if (dvr_entry_get_removal_days(de) != DVR_RET_SPACENEED) // only remove the allowed ones
if (dvr_entry_get_removal_days(de) != DVR_RET_SPACE) // only remove the allowed ones
continue;

if (dvr_get_filename(de) == NULL || dvr_get_filesize(de) <= 0)
Expand All @@ -124,6 +131,7 @@ dvr_disk_space_cleanup(dvr_config_t *cfg)
fileSize = dvr_get_filesize(oldest);
availBytes += fileSize;
clearedBytes += fileSize;
usedBytes -= fileSize;

localtime_r(&stoptime, &tm);
if (strftime(tbuf, sizeof(tbuf), "%F %T", &tm) <= 0)
Expand All @@ -149,8 +157,8 @@ dvr_disk_space_cleanup(dvr_config_t *cfg)
}

finish:
tvhlog(LOG_INFO, "dvr","disk space cleanup for config \"%s\", cleared \"%ld MB\" of disk space, new free disk space \"%ld MB\"",
configName, clearedBytes/(int64_t)1024/(int64_t)1024, availBytes/(int64_t)1024/(int64_t)1024);
tvhlog(LOG_INFO, "dvr","disk space cleanup for config \"%s\", cleared \"%ld MB\" of disk space, new free disk space \"%ld MiB\", new used disk space \"%ld MiB\"",
configName, TOMIB(clearedBytes), TOMIB(availBytes), TOMIB(usedBytes));

return clearedBytes;
}
Expand All @@ -165,7 +173,7 @@ dvr_disk_space_check()
dvr_config_t *cfg;
dvr_entry_t *de;
struct statvfs diskdata;
int64_t requiredBytes, availBytes;
int64_t requiredBytes, maximalBytes, availBytes, usedBytes;
int idx = 0, cleanupDone = 0;

pthread_mutex_lock(&global_lock);
Expand All @@ -183,29 +191,37 @@ dvr_disk_space_check()
statvfs(cfg->dvr_storage, &diskdata) != -1)
{
availBytes = diskdata.f_bsize * (int64_t)diskdata.f_bavail;
requiredBytes = (int64_t)cfg->dvr_cleanup_threshold*(int64_t)1024*(int64_t)1024;
usedBytes = (diskdata.f_bsize * (int64_t)diskdata.f_blocks) - availBytes;
requiredBytes = MIB(cfg->dvr_cleanup_threshold_low);
maximalBytes = MIB(cfg->dvr_cleanup_threshold_high);

if (availBytes < requiredBytes) {
if (availBytes < requiredBytes || maximalBytes > usedBytes) {
LIST_FOREACH(de, &dvrentries, de_global_link) {

/* only start cleanup if we are actually writing files right now */
if (de->de_sched_state != DVR_RECORDING || !de->de_config || de->de_config != cfg)
continue;

tvhlog(LOG_WARNING, "dvr","running out of free disk space for dvr config \"%s\", required free space \"%ld MB\", current free space \"%ld MB\"",
cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile",
requiredBytes/(int64_t)1024/(int64_t)1024, availBytes/(int64_t)1024/(int64_t)1024);
goto checking;

if (availBytes < requiredBytes) {
tvhlog(LOG_WARNING, "dvr","running out of free disk space for dvr config \"%s\", required free space \"%ld MiB\", current free space \"%ld MiB\"",
cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile",
TOMIB(requiredBytes), TOMIB(availBytes));
} else {
tvhlog(LOG_WARNING, "dvr","running out of used disk space for dvr config \"%s\", required used space \"%ld MiB\", current used space \"%ld MiB\"",
cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile",
TOMIB(maximalBytes), TOMIB(usedBytes));
}

/* only cleanup one directory at the time as the system needs time to delete the actual files */
dvr_disk_space_cleanup(de->de_config);
cleanupDone = 1;
dvr_disk_space_config_idx = idx + 1;
break;
}
}
else {
tvhlog(LOG_DEBUG, "dvr","checking free disk space for config \"%s\" : OK",
cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile");
} else {
checking:
tvhlog(LOG_DEBUG, "dvr","checking free and used disk space for config \"%s\" : OK",
cfg != dvr_config_find_by_name(NULL) ? cfg->dvr_config_name : "Default profile");
}
}
}
Expand Down

0 comments on commit 18a4a8c

Please sign in to comment.