From 87ca75098cf5445131fdaf565fe13183528d1b52 Mon Sep 17 00:00:00 2001 From: Pantelis Roditis Date: Thu, 3 Dec 2015 15:14:26 +0200 Subject: [PATCH 1/2] Introduce event for archive rotation of logs. --- schema/echofish-dataonly.sql | 6 +++++- schema/echofish-events.sql | 42 +++++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/schema/echofish-dataonly.sql b/schema/echofish-dataonly.sql index 09f4eaa..68415c4 100644 --- a/schema/echofish-dataonly.sql +++ b/schema/echofish-dataonly.sql @@ -24,7 +24,11 @@ SET time_zone = "+00:00"; -- Dumping data for table `sysconf` -- insert into sysconf (id,val) VALUES ('archive_activated','yes'), - ('whitelist_archived','no') + ('whitelist_archived','no'), + ('archive_rotate','yes'), + ('archive_delete_use_mem','no'), + ('archive_delete_days',7), + ('archive_delete_limit', 0) ON DUPLICATE KEY UPDATE id=VALUES(id); -- diff --git a/schema/echofish-events.sql b/schema/echofish-events.sql index e3c81d6..8cfa584 100644 --- a/schema/echofish-events.sql +++ b/schema/echofish-events.sql @@ -3,7 +3,7 @@ SET FOREIGN_KEY_CHECKS=0; SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO"; SET time_zone = "+00:00"; - +SET NAMES utf8 COLLATE 'utf8_unicode_ci'; /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; @@ -32,3 +32,43 @@ BEGIN ALTER EVENT e_archive_parse_unparsed ENABLE; END // + +DROP EVENT IF EXISTS e_rotate_archive// +CREATE EVENT e_rotate_archive +ON SCHEDULE EVERY 1 DAY COMMENT 'ROTATE OLD ARCHIVE ENTRIES' DO +BEGIN +IF (SELECT count(*) FROM sysconf WHERE id='archive_rotate' and val='yes')>0 THEN + SET @archive_days=IFNULL((SELECT val FROM sysconf WHERE id='archive_delete_days'),7); + SET @archive_limit=IFNULL((SELECT val FROM sysconf WHERE id='archive_delete_limit'),0); + SET @use_mem=IFNULL((SELECT val FROM sysconf WHERE id='archive_delete_use_mem'),'no'); + IF @archive_days>0 THEN + IF @use_mem != 'yes' THEN + CREATE TEMPORARY TABLE archive_ids (id BIGINT UNSIGNED NOT NULL PRIMARY KEY); + ELSE + CREATE TEMPORARY TABLE archive_ids (id BIGINT UNSIGNED NOT NULL PRIMARY KEY) ENGINE=MEMORY; + END IF; + + SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + START TRANSACTION; + IF @archive_limit > 0 THEN + PREPARE choose_archive_ids FROM 'INSERT INTO archive_ids SELECT id FROM `archive` WHERE received_ts < NOW() - INTERVAL ? DAY LIMIT ?'; + EXECUTE choose_archive_ids USING @archive_days, @archive_limit; + ELSE + PREPARE choose_archive_ids FROM 'INSERT INTO archive_ids SELECT id FROM `archive` WHERE received_ts < NOW() - INTERVAL ?'; + EXECUTE choose_archive_ids USING @archive_days; + END IF; + DEALLOCATE PREPARE choose_archive_ids; + -- Ignore ID's from entries that exist on archive_unparse + DELETE t1.* FROM archive_ids as t1 LEFT JOIN archive_unparse AS t2 ON t1.id=t2.id WHERE t2.id IS NOT NULL; + -- Ignore ID's from entries that exist on syslog + DELETE t1.* FROM archive_ids as t1 LEFT JOIN syslog AS t2 ON t1.id=t2.id WHERE t2.id IS NOT NULL; + -- Ignore ID's from entries that exist on abuser_evidense + DELETE t1.* FROM archive_ids as t1 LEFT JOIN abuser_evidence AS t2 ON t1.id=t2.archive_id WHERE t2.archive_id IS NOT NULL; + DELETE t1.* FROM `archive` AS t1 LEFT JOIN archive_ids AS t2 ON t1.id=t2.id WHERE t2.id IS NOT NULL; + COMMIT; + END IF; +END IF; +END +// + +ALTER EVENT e_rotate_archive DISABLE// From a5a7262998f3a530407a4dbcc47738a068c77eda Mon Sep 17 00:00:00 2001 From: Pantelis Roditis Date: Fri, 4 Dec 2015 11:41:40 +0200 Subject: [PATCH 2/2] seems that prev commit got lost in the git limbo --- schema/echofish-dataonly.sql | 2 +- schema/echofish-events.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/echofish-dataonly.sql b/schema/echofish-dataonly.sql index 68415c4..5dbf776 100644 --- a/schema/echofish-dataonly.sql +++ b/schema/echofish-dataonly.sql @@ -28,7 +28,7 @@ insert into sysconf (id,val) VALUES ('archive_activated','yes'), ('archive_rotate','yes'), ('archive_delete_use_mem','no'), ('archive_delete_days',7), - ('archive_delete_limit', 0) + ('archive_delete_limit',0) ON DUPLICATE KEY UPDATE id=VALUES(id); -- diff --git a/schema/echofish-events.sql b/schema/echofish-events.sql index 8cfa584..28ed387 100644 --- a/schema/echofish-events.sql +++ b/schema/echofish-events.sql @@ -71,4 +71,4 @@ END IF; END // -ALTER EVENT e_rotate_archive DISABLE// +