Skip to content

Commit

Permalink
Hashing workaround for unique indexes (#389)
Browse files Browse the repository at this point in the history 
* hashing workaround on structure page

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* Create hash columns and unique indexes on upgrade

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* fix syntax

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* calculate hash values using md5

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* Change binary size to 16

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* remove linktrack

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* apply md5 to regex new entries

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* update queries for url new entries

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* apply changes to sendmaillib

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* change datatype and add where clause

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* apply changes to structure

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>

* fix typos

Signed-off-by: Xheni Myrtaj <myrtajxheni@gmail.com>
  • Loading branch information
@xh3n1 @michield
xh3n1 authored and michield committed Aug 22, 2018
1 parent d1d6ce3 commit 56c0394
Show file tree
Hide file tree
Showing 6 changed files with 39 additions and 29 deletions.
4 changes: 2 additions & 2 deletions public_html/lists/admin/bouncerule.php
View file
Expand Up @@ -30,8 +30,8 @@
}

if (isset($_POST['save']) && $_POST['save']) {
Sql_Query(sprintf('update %s set regex = "%s",action="%s", comment="%s",status = "%s" where id= %d',
$GLOBALS['tables']['bounceregex'], trim($_POST['regex']), sql_escape($_POST['action']),
Sql_Query(sprintf('update %s set regex = "%s", regexhash = "%s", action="%s", comment="%s",status = "%s" where id= %d',
$GLOBALS['tables']['bounceregex'], trim($_POST['regex']), md5(trim($_POST['regex'])), sql_escape($_POST['action']),
sql_escape($_POST['comment']), sql_escape($_POST['status']), $_GET['id']), 1);
$num = Sql_Affected_Rows();
if ($num < 0) {
Expand Down
4 changes: 2 additions & 2 deletions public_html/lists/admin/bouncerules.php
View file
Expand Up @@ -46,8 +46,8 @@
}

if (isset($_POST['newrule']) && $_POST['newrule']) {
Sql_Query(sprintf('insert into %s (regex,action,comment,admin,status) values("%s","%s","%s",%d,"active")',
$GLOBALS['tables']['bounceregex'], sql_escape($_POST['newrule']), sql_escape($_POST['action']),
Sql_Query(sprintf('insert into %s (regex, regexhash, action,comment,admin,status) values("%s","%s","%s","%s",%d,"active")',
$GLOBALS['tables']['bounceregex'], sql_escape($_POST['newrule']), md5(sql_escape($_POST['newrule'])), sql_escape($_POST['action']),
sql_escape($_POST['comment']), $_SESSION['logindetails']['id']), 1);
$num = Sql_Affected_Rows();
if ($num < 0) {
Expand Down
8 changes: 4 additions & 4 deletions public_html/lists/admin/convertstats.php
View file
Expand Up @@ -83,12 +83,12 @@ function flushbuffer()
break;
}

$exists = Sql_Fetch_Row_Query(sprintf('select id from %s where url = "%s"', $GLOBALS['tables']['linktrack_forward'],
$row['url']));
$exists = Sql_Fetch_Row_Query(sprintf('select id from %s where urlhash = "%s"', $GLOBALS['tables']['linktrack_forward'],
md5($row['url'])));
if (!$exists[0]) {
$personalise = preg_match('/uid=/', $row['forward']);
Sql_Query(sprintf('insert into %s (url,personalise) values("%s",%d)', $GLOBALS['tables']['linktrack_forward'],
$row['url'], $personalise));
Sql_Query(sprintf('insert into %s (url, urlhash, personalise) values("%s","%s", %d)', $GLOBALS['tables']['linktrack_forward'],
$row['url'], md5($row['url']), $personalise));
$fwdid = Sql_Insert_id();
} else {
$fwdid = $exists[0];
Expand Down
8 changes: 4 additions & 4 deletions public_html/lists/admin/sendemaillib.php
View file
Expand Up @@ -1263,13 +1263,13 @@ function clickTrackLinkId($messageid, $userid, $url, $link)
* alter table phplist_linktrack_forward add index (url);
* */

$exists = Sql_Fetch_Row_Query(sprintf('select id,uuid from %s where url = "%s"',
$GLOBALS['tables']['linktrack_forward'], sql_escape(substr($url, 0, 255))));
$exists = Sql_Fetch_Row_Query(sprintf('select id,uuid from %s where urlhash = "%s"',
$GLOBALS['tables']['linktrack_forward'], md5(sql_escape($url))));
if (empty($exists[0])) {
$personalise = preg_match('/uid=/', $link);
$uuid = (string)Uuid::generate(4);
Sql_Query(sprintf('insert into %s set url = "%s", personalise = %d, uuid = "%s"',
$GLOBALS['tables']['linktrack_forward'], sql_escape($url), $personalise, $uuid));
Sql_Query(sprintf('insert into %s set url = "%s", urlhash = "%s", personalise = %d, uuid = "%s"',
$GLOBALS['tables']['linktrack_forward'], sql_escape($url), md5(sql_escape($url)), $personalise, $uuid));
$fwdid = Sql_Insert_id();
$fwduuid = $uuid;
} elseif (empty($exists[1])) {
Expand Down
21 changes: 12 additions & 9 deletions public_html/lists/admin/structure.php
View file
Expand Up @@ -359,7 +359,7 @@
),
'urlcache' => array(
'id' => array('integer not null primary key auto_increment', 'ID'),
'url' => array('text not null', ''),
'url' => array('varchar(2083) not null', ''),
'lastmodified' => array('integer', ''),
'added' => array('datetime', ''),
'content' => array('longblob', ''),
Expand All @@ -376,21 +376,22 @@

//# keep it in for now, but could be dropped at some point
//# once all dependencies have been removed
// tables that have unique indexes on hash values : linktrack_forward, bounceregex
'linktrack' => array(
'linkid' => array('integer not null primary key auto_increment', 'Link ID'),
'messageid' => array('integer not null', 'Message ID'),
'userid' => array('integer not null', 'subscriber ID'),
'url' => array('text', 'URL to log'),
'forward' => array('text', 'URL to forward to'),
'url' => array('varchar(255)', 'URL to log'),
'forward' => array('varchar(255)', 'URL to forward to'),
'firstclick' => array('datetime', 'When first clicked'),
'latestclick' => array('timestamp', 'When last clicked'),
'clicked' => array('integer default 0', 'Number of clicks'),
'index_1' => array('midindex (messageid)', ''),
'index_2' => array('uidindex (userid)', ''),
'index_3' => array('urlindex (url(255))', ''),
'index_3' => array('urlindex (url)', ''),
'index_4' => array('miduidindex (messageid,userid)', ''),
// "index_5" => array("miduidurlindex (messageid,userid,url)",""),
'unique_1' => array('miduidurlindex (messageid,userid,url(255))', ''),
'unique_1' => array('miduidurlindex (messageid,userid,url)', ''),
),
'linktrack_ml' => array( // ml -> message link
'messageid' => array('integer not null', 'Message ID'),
Expand Down Expand Up @@ -425,14 +426,15 @@
),
'linktrack_forward' => array(
'id' => array('integer not null primary key auto_increment', 'forward ID'),
'url' => array('text', 'URL to log'),
'url' => array('varchar(2083)', 'URL to log'),
'urlhash' => array('char(32)', 'hash value of URL'),
'uuid' => array('varchar(36) default ""', 'UUID'),
// "forward" => array("text","URL to forward to"),
'personalise' => array('tinyint default 0', 'Forward adding the UID?'),
'index_1' => array('urlindex (url(255))', ''),
// "index_2" => array("urlforwardindex (url,forward(255))",""),
// 'unique_1' => array('fwdunique (forward (500))','Forward should be unique'),
'unique_1' => array('urlunique (url(255))', 'URL should be unique'),
'unique_1' => array('urlunique (urlhash)', 'URL should be unique'),
'index_2' => array('uuididx (uuid)', 'sys:index'),
),
'linktrack_userclick' => array(
Expand Down Expand Up @@ -468,14 +470,15 @@
// ),
'bounceregex' => array(
'id' => array('integer not null primary key auto_increment', 'ID'),
'regex' => array('text', 'Regex'),
'regex' => array('varchar(2083)', 'Regex'),
'regexhash' => array('char(32)', 'hash value of regex'),
'action' => array('varchar(255)', 'Action on rule'),
'listorder' => array('integer default 0', ''),
'admin' => array('integer', ''),
'comment' => array('text', ''),
'status' => array('varchar(255)', ''),
'count' => array('integer default 0', 'Count of matching bounces on this rule'),
'unique_1' => array('regex (regex(255))', ''),
'unique_1' => array('regex (regexhash)', ''),
),
'bounceregex_bounce' => array(
'regex' => array('integer not null', 'Related regex'),
Expand Down
23 changes: 15 additions & 8 deletions public_html/lists/admin/upgrade.php
View file
Expand Up @@ -118,26 +118,33 @@ function output($message)
'urlcache' => array(
'urlindex' => array('value' => 'url(255)', 'unique' => false),
),
'linktrack' => array(
'urlindex' => array('value' => 'url(255)', 'unique' => false),
'miduidurlindex' => array('value' => 'messageid,userid,url(255)', 'unique' => true),
),
'linktrack_forward' => array(
'urlindex' => array('value' => 'url(255)', 'unique' => false),
'urlunique' => array('value' => 'url(255)', 'unique' => true),
'urlunique' => array('value' => 'urlhash', 'unique' => true),
),
'bounceregex' => array(
'regex' => array('value' => 'regex(255)', 'unique'=> true),
'regex' => array('value' => 'regexhash', 'unique'=> true),
),
);

$tablesToAlter = array(
'urlcache' => array('url'),
'linktrack' => array('url', 'forward'),
'linktrack_forward' => array('url'),
'bounceregex' => array('regex'),
);

//add columns for hash values

Sql_Query("alter table {$GLOBALS['tables']['linktrack_forward']} add urlhash char(32) ");
Sql_Query("alter table {$GLOBALS['tables']['bounceregex']} add regexhash char(32) ");

// add hash values

Sql_Query("update {$GLOBALS['tables']['linktrack_forward']} set urlhash = md5(url) where urlhash is NULL ");
Sql_Query("update {$GLOBALS['tables']['bounceregex']} set regexhash = md5(regex) where regexhash is NULL ");



foreach($indexesToRecreate as $table => $indexes) {


Expand All @@ -151,7 +158,7 @@ function output($message)

$alteringOperations = $tablesToAlter[$table];
foreach($alteringOperations as $operation) {
Sql_Query("alter table {$GLOBALS['tables'][$table]} modify $operation text ");
Sql_Query("alter table {$GLOBALS['tables'][$table]} modify $operation varchar(2083) ");
}

foreach($indexes as $indexName => $settings) {
Expand Down

0 comments on commit 56c0394

Please sign in to comment.