Fixed issue: Participant import with duplicate check very slow when t…

…here more than 100k participants
LimeSurvey · Sep 20, 2020 · 00d3a31 · 00d3a31 · Shnoulle · Sep 21, 2020
1 parent cab8d3b
commit 00d3a31
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 3 deletions.
diff --git a/application/config/version.php b/application/config/version.php
@@ -12,7 +12,7 @@
  */
 
 $config['versionnumber'] = '4.3.16';
-$config['dbversionnumber'] = 427;
+$config['dbversionnumber'] = 428;
 $config['buildnumber'] = '';
 $config['updatable'] = true;
 $config['templateapiversion']  = 3;

diff --git a/application/helpers/update/updatedb_helper.php b/application/helpers/update/updatedb_helper.php
@@ -2931,6 +2931,36 @@ function db_upgrade_all($iOldDBVersion, $bSilent = false)
             $oDB->createCommand()->update('{{settings_global}}',array('stg_value'=>420),"stg_name='DBVersion'");
             $oTransaction->commit();
         }
+
+        /*
+        * DBVersion 361 & 362 were intentionally left out to sync with Cloud Hosting
+        */
+
+        /*
+         * Correct permission for survey menu Survey Participants (tokens, not surveysettings).
+         */
+        if ($iOldDBVersion < 363) {
+            $oTransaction = $oDB->beginTransaction();
+            $aTableNames = dbGetTablesLike("tokens%");
+            $oDB = Yii::app()->getDb();
+            foreach ($aTableNames as $sTableName) {
+                try { 
+                        setTransactionBookmark(); 
+                        switch (Yii::app()->db->driverName){
+                            case 'mysql':
+                            case 'mysqli':
+                                $oDB->createCommand()->createIndex('idx_email', $sTableName, 'email(30)', false);
+                                break;
+                            case 'pgsql':
+                                $oDB->createCommand()->createIndex('idx_email', $sTableName, 'email', false);
+                                break;
+                            // MSSQL does not support indexes on text fields so no dice
+                        }
+                    } catch (Exception $e) { rollBackToTransactionBookmark(); }
+            }
+            $oDB->createCommand()->update('{{settings_global}}', ['stg_value'=>363], "stg_name='DBVersion'");
+            $oTransaction->commit();
+        }        
 
         if($iOldDBVersion < 421) {
             $oTransaction = $oDB->beginTransaction();
@@ -3109,6 +3139,34 @@ function db_upgrade_all($iOldDBVersion, $bSilent = false)
             $oDB->createCommand()->update('{{settings_global}}', array('stg_value' => 427), "stg_name='DBVersion'");
             $oTransaction->commit();
         }
+
+        /*
+         * Correct permission for survey menu Survey Participants (tokens, not surveysettings).
+         */
+        if ($iOldDBVersion < 428) {
+            $oTransaction = $oDB->beginTransaction();
+            $aTableNames = dbGetTablesLike("tokens%");
+            $oDB = Yii::app()->getDb();
+            foreach ($aTableNames as $sTableName) {
+                try { 
+                        setTransactionBookmark(); 
+                        switch (Yii::app()->db->driverName){
+                            case 'mysql':
+                            case 'mysqli':
+                                $oDB->createCommand()->createIndex('idx_email', $sTableName, 'email(30)', false);
+                                break;
+                            case 'pgsql':
+                                $oDB->createCommand()->createIndex('idx_email', $sTableName, 'email', false);
+                                break;
+                            // MSSQL does not support indexes on text fields so no dice
+                        }
+                    } catch (Exception $e) { rollBackToTransactionBookmark(); }
+            }
+            $oDB->createCommand()->update('{{settings_global}}', ['stg_value'=>428], "stg_name='DBVersion'");
+            $oTransaction->commit();
+        }   
+
+
     } catch (Exception $e) {
         Yii::app()->setConfig('Updating', false);
         $oTransaction->rollback();

diff --git a/application/models/Token.php b/application/models/Token.php
@@ -199,10 +199,22 @@ public static function createTable($surveyId, array $extraFields = array())
         $db->createCommand()->createTable($sTableName, $fields, $options);
 
         /**
-         * @todo Check if this random component in the index name is needed.
-         * As far as I (sam) know index names need only be unique per table.
+         * The random component in the index name is needed because MSSQL is being the dorky kid and 
+         * complaining about duplicates when renaming the table and trying to use the same index again 
+         * on a new token table (for example on reactivation)
          */
         $db->createCommand()->createIndex("idx_token_token_{$surveyId}_".rand(1, 50000), $sTableName, 'token');
+
+        // MSSQL does not support indexes on text fields so not needed here
+        switch (Yii::app()->db->driverName){
+            case 'mysql':
+            case 'mysqli':
+                $db->createCommand()->createIndex('idx_email', $sTableName, 'email(30)', false);
+                break;
+            case 'pgsql':
+                $db->createCommand()->createIndex('idx_email', $sTableName, 'email', false);
+                break;
+        }
 
         // Refresh schema cache just in case the table existed in the past, and return if table exist
         return $db->schema->getTable($sTableName, true);