From 56c039422885948a19178e6e1fe890aea2704f89 Mon Sep 17 00:00:00 2001 From: Xheni Myrtaj Date: Wed, 22 Aug 2018 14:49:38 +0200 Subject: [PATCH] Hashing workaround for unique indexes (#389) * hashing workaround on structure page Signed-off-by: Xheni Myrtaj * Create hash columns and unique indexes on upgrade Signed-off-by: Xheni Myrtaj * fix syntax Signed-off-by: Xheni Myrtaj * calculate hash values using md5 Signed-off-by: Xheni Myrtaj * Change binary size to 16 Signed-off-by: Xheni Myrtaj * remove linktrack Signed-off-by: Xheni Myrtaj * apply md5 to regex new entries Signed-off-by: Xheni Myrtaj * update queries for url new entries Signed-off-by: Xheni Myrtaj * apply changes to sendmaillib Signed-off-by: Xheni Myrtaj * change datatype and add where clause Signed-off-by: Xheni Myrtaj * apply changes to structure Signed-off-by: Xheni Myrtaj * fix typos Signed-off-by: Xheni Myrtaj --- public_html/lists/admin/bouncerule.php | 4 ++-- public_html/lists/admin/bouncerules.php | 4 ++-- public_html/lists/admin/convertstats.php | 8 ++++---- public_html/lists/admin/sendemaillib.php | 8 ++++---- public_html/lists/admin/structure.php | 21 ++++++++++++--------- public_html/lists/admin/upgrade.php | 23 +++++++++++++++-------- 6 files changed, 39 insertions(+), 29 deletions(-) diff --git a/public_html/lists/admin/bouncerule.php b/public_html/lists/admin/bouncerule.php index 42d7cd160..6e1114376 100644 --- a/public_html/lists/admin/bouncerule.php +++ b/public_html/lists/admin/bouncerule.php @@ -30,8 +30,8 @@ } if (isset($_POST['save']) && $_POST['save']) { - Sql_Query(sprintf('update %s set regex = "%s",action="%s", comment="%s",status = "%s" where id= %d', - $GLOBALS['tables']['bounceregex'], trim($_POST['regex']), sql_escape($_POST['action']), + Sql_Query(sprintf('update %s set regex = "%s", regexhash = "%s", action="%s", comment="%s",status = "%s" where id= %d', + $GLOBALS['tables']['bounceregex'], trim($_POST['regex']), md5(trim($_POST['regex'])), sql_escape($_POST['action']), sql_escape($_POST['comment']), sql_escape($_POST['status']), $_GET['id']), 1); $num = Sql_Affected_Rows(); if ($num < 0) { diff --git a/public_html/lists/admin/bouncerules.php b/public_html/lists/admin/bouncerules.php index 3eb7889b4..8e8423350 100644 --- a/public_html/lists/admin/bouncerules.php +++ b/public_html/lists/admin/bouncerules.php @@ -46,8 +46,8 @@ } if (isset($_POST['newrule']) && $_POST['newrule']) { - Sql_Query(sprintf('insert into %s (regex,action,comment,admin,status) values("%s","%s","%s",%d,"active")', - $GLOBALS['tables']['bounceregex'], sql_escape($_POST['newrule']), sql_escape($_POST['action']), + Sql_Query(sprintf('insert into %s (regex, regexhash, action,comment,admin,status) values("%s","%s","%s","%s",%d,"active")', + $GLOBALS['tables']['bounceregex'], sql_escape($_POST['newrule']), md5(sql_escape($_POST['newrule'])), sql_escape($_POST['action']), sql_escape($_POST['comment']), $_SESSION['logindetails']['id']), 1); $num = Sql_Affected_Rows(); if ($num < 0) { diff --git a/public_html/lists/admin/convertstats.php b/public_html/lists/admin/convertstats.php index f38b6a7aa..07cbbe108 100644 --- a/public_html/lists/admin/convertstats.php +++ b/public_html/lists/admin/convertstats.php @@ -83,12 +83,12 @@ function flushbuffer() break; } - $exists = Sql_Fetch_Row_Query(sprintf('select id from %s where url = "%s"', $GLOBALS['tables']['linktrack_forward'], - $row['url'])); + $exists = Sql_Fetch_Row_Query(sprintf('select id from %s where urlhash = "%s"', $GLOBALS['tables']['linktrack_forward'], + md5($row['url']))); if (!$exists[0]) { $personalise = preg_match('/uid=/', $row['forward']); - Sql_Query(sprintf('insert into %s (url,personalise) values("%s",%d)', $GLOBALS['tables']['linktrack_forward'], - $row['url'], $personalise)); + Sql_Query(sprintf('insert into %s (url, urlhash, personalise) values("%s","%s", %d)', $GLOBALS['tables']['linktrack_forward'], + $row['url'], md5($row['url']), $personalise)); $fwdid = Sql_Insert_id(); } else { $fwdid = $exists[0]; diff --git a/public_html/lists/admin/sendemaillib.php b/public_html/lists/admin/sendemaillib.php index 4d65d0b3f..15385bcde 100644 --- a/public_html/lists/admin/sendemaillib.php +++ b/public_html/lists/admin/sendemaillib.php @@ -1263,13 +1263,13 @@ function clickTrackLinkId($messageid, $userid, $url, $link) * alter table phplist_linktrack_forward add index (url); * */ - $exists = Sql_Fetch_Row_Query(sprintf('select id,uuid from %s where url = "%s"', - $GLOBALS['tables']['linktrack_forward'], sql_escape(substr($url, 0, 255)))); + $exists = Sql_Fetch_Row_Query(sprintf('select id,uuid from %s where urlhash = "%s"', + $GLOBALS['tables']['linktrack_forward'], md5(sql_escape($url)))); if (empty($exists[0])) { $personalise = preg_match('/uid=/', $link); $uuid = (string)Uuid::generate(4); - Sql_Query(sprintf('insert into %s set url = "%s", personalise = %d, uuid = "%s"', - $GLOBALS['tables']['linktrack_forward'], sql_escape($url), $personalise, $uuid)); + Sql_Query(sprintf('insert into %s set url = "%s", urlhash = "%s", personalise = %d, uuid = "%s"', + $GLOBALS['tables']['linktrack_forward'], sql_escape($url), md5(sql_escape($url)), $personalise, $uuid)); $fwdid = Sql_Insert_id(); $fwduuid = $uuid; } elseif (empty($exists[1])) { diff --git a/public_html/lists/admin/structure.php b/public_html/lists/admin/structure.php index 9799aa786..4ae9f1c25 100644 --- a/public_html/lists/admin/structure.php +++ b/public_html/lists/admin/structure.php @@ -359,7 +359,7 @@ ), 'urlcache' => array( 'id' => array('integer not null primary key auto_increment', 'ID'), - 'url' => array('text not null', ''), + 'url' => array('varchar(2083) not null', ''), 'lastmodified' => array('integer', ''), 'added' => array('datetime', ''), 'content' => array('longblob', ''), @@ -376,21 +376,22 @@ //# keep it in for now, but could be dropped at some point //# once all dependencies have been removed + // tables that have unique indexes on hash values : linktrack_forward, bounceregex 'linktrack' => array( 'linkid' => array('integer not null primary key auto_increment', 'Link ID'), 'messageid' => array('integer not null', 'Message ID'), 'userid' => array('integer not null', 'subscriber ID'), - 'url' => array('text', 'URL to log'), - 'forward' => array('text', 'URL to forward to'), + 'url' => array('varchar(255)', 'URL to log'), + 'forward' => array('varchar(255)', 'URL to forward to'), 'firstclick' => array('datetime', 'When first clicked'), 'latestclick' => array('timestamp', 'When last clicked'), 'clicked' => array('integer default 0', 'Number of clicks'), 'index_1' => array('midindex (messageid)', ''), 'index_2' => array('uidindex (userid)', ''), - 'index_3' => array('urlindex (url(255))', ''), + 'index_3' => array('urlindex (url)', ''), 'index_4' => array('miduidindex (messageid,userid)', ''), // "index_5" => array("miduidurlindex (messageid,userid,url)",""), - 'unique_1' => array('miduidurlindex (messageid,userid,url(255))', ''), + 'unique_1' => array('miduidurlindex (messageid,userid,url)', ''), ), 'linktrack_ml' => array( // ml -> message link 'messageid' => array('integer not null', 'Message ID'), @@ -425,14 +426,15 @@ ), 'linktrack_forward' => array( 'id' => array('integer not null primary key auto_increment', 'forward ID'), - 'url' => array('text', 'URL to log'), + 'url' => array('varchar(2083)', 'URL to log'), + 'urlhash' => array('char(32)', 'hash value of URL'), 'uuid' => array('varchar(36) default ""', 'UUID'), // "forward" => array("text","URL to forward to"), 'personalise' => array('tinyint default 0', 'Forward adding the UID?'), 'index_1' => array('urlindex (url(255))', ''), // "index_2" => array("urlforwardindex (url,forward(255))",""), // 'unique_1' => array('fwdunique (forward (500))','Forward should be unique'), - 'unique_1' => array('urlunique (url(255))', 'URL should be unique'), + 'unique_1' => array('urlunique (urlhash)', 'URL should be unique'), 'index_2' => array('uuididx (uuid)', 'sys:index'), ), 'linktrack_userclick' => array( @@ -468,14 +470,15 @@ // ), 'bounceregex' => array( 'id' => array('integer not null primary key auto_increment', 'ID'), - 'regex' => array('text', 'Regex'), + 'regex' => array('varchar(2083)', 'Regex'), + 'regexhash' => array('char(32)', 'hash value of regex'), 'action' => array('varchar(255)', 'Action on rule'), 'listorder' => array('integer default 0', ''), 'admin' => array('integer', ''), 'comment' => array('text', ''), 'status' => array('varchar(255)', ''), 'count' => array('integer default 0', 'Count of matching bounces on this rule'), - 'unique_1' => array('regex (regex(255))', ''), + 'unique_1' => array('regex (regexhash)', ''), ), 'bounceregex_bounce' => array( 'regex' => array('integer not null', 'Related regex'), diff --git a/public_html/lists/admin/upgrade.php b/public_html/lists/admin/upgrade.php index 407cd3c5e..462192150 100644 --- a/public_html/lists/admin/upgrade.php +++ b/public_html/lists/admin/upgrade.php @@ -118,26 +118,33 @@ function output($message) 'urlcache' => array( 'urlindex' => array('value' => 'url(255)', 'unique' => false), ), - 'linktrack' => array( - 'urlindex' => array('value' => 'url(255)', 'unique' => false), - 'miduidurlindex' => array('value' => 'messageid,userid,url(255)', 'unique' => true), - ), 'linktrack_forward' => array( 'urlindex' => array('value' => 'url(255)', 'unique' => false), - 'urlunique' => array('value' => 'url(255)', 'unique' => true), + 'urlunique' => array('value' => 'urlhash', 'unique' => true), ), 'bounceregex' => array( - 'regex' => array('value' => 'regex(255)', 'unique'=> true), + 'regex' => array('value' => 'regexhash', 'unique'=> true), ), ); $tablesToAlter = array( 'urlcache' => array('url'), - 'linktrack' => array('url', 'forward'), 'linktrack_forward' => array('url'), 'bounceregex' => array('regex'), ); + //add columns for hash values + + Sql_Query("alter table {$GLOBALS['tables']['linktrack_forward']} add urlhash char(32) "); + Sql_Query("alter table {$GLOBALS['tables']['bounceregex']} add regexhash char(32) "); + + // add hash values + + Sql_Query("update {$GLOBALS['tables']['linktrack_forward']} set urlhash = md5(url) where urlhash is NULL "); + Sql_Query("update {$GLOBALS['tables']['bounceregex']} set regexhash = md5(regex) where regexhash is NULL "); + + + foreach($indexesToRecreate as $table => $indexes) { @@ -151,7 +158,7 @@ function output($message) $alteringOperations = $tablesToAlter[$table]; foreach($alteringOperations as $operation) { - Sql_Query("alter table {$GLOBALS['tables'][$table]} modify $operation text "); + Sql_Query("alter table {$GLOBALS['tables'][$table]} modify $operation varchar(2083) "); } foreach($indexes as $indexName => $settings) {