Skip to content

Commit

Permalink
Fixed issue: Integrity check very slow when having a lot of surveys (#…
Browse files Browse the repository at this point in the history 
…3744)

Dev There is no way to detach the very costly event handler AfterFindSurvey from a model so a new model SurveyNoEvent was introduced for the special cases of using an activeRecord model for Survey->findAll
Dev Conditioned check for correct response table fields. This is now only run if the URL parameter checkResponseTableFields== y is set
  • Loading branch information
@c-schmitz
c-schmitz committed Feb 16, 2024
1 parent a089c46 commit ad0a1d6
Show file tree
Hide file tree
Showing 7 changed files with 328 additions and 132 deletions.
202 changes: 98 additions & 104 deletions application/controllers/admin/CheckIntegrity.php
View file
Expand Up @@ -606,7 +606,8 @@ protected function checkintegrity()
}

// Deactivate surveys that have a missing response table
$oSurveys = Survey::model()->findAll(array('order' => 'sid'));
$survey = new SurveyLight();
$oSurveys = $survey->findAll(array('order' => 'sid'));
$oDB = Yii::app()->getDb();
$oDB->schemaCachingDuration = 0; // Deactivate schema caching
Yii::app()->setConfig('Updating', true);
Expand All @@ -618,83 +619,89 @@ protected function checkintegrity()
}
}

/** Check for active surveys if questions are in the correct group **/
foreach ($oSurveys as $oSurvey) {
// This actually clears the schema cache, not just refreshes it
$oDB->schema->refresh();
// We get the active surveys
if ($oSurvey->isActive && $oSurvey->hasResponsesTable) {
$model = SurveyDynamic::model($oSurvey->sid);
$aColumns = $model->getMetaData()->columns;
$aQids = array();

// We get the columns of the responses table
foreach ($aColumns as $oColumn) {
// Question columns start with the SID
if (strpos((string) $oColumn->name, (string)$oSurvey->sid) !== false) {
// Fileds are separated by X
$aFields = explode('X', (string) $oColumn->name);

if (isset($aFields[1])) {
$sGid = $aFields[1];

// QID field can be more than just QID, like: 886other or 886A1
// So we clean it by finding the first alphabetical character
$sDirtyQid = $aFields[2];
preg_match('~[a-zA-Z_#]~i', $sDirtyQid, $match, PREG_OFFSET_CAPTURE);

if (isset($match[0][1])) {
$sQID = substr($sDirtyQid, 0, $match[0][1]);
} else {
// It was just the QID.... (maybe)
$sQID = $sDirtyQid;
}
/**
* Check for active surveys if questions are in the correct group
* This will only run if an additional URL parameter checkResponseTableFields=y is set
* This is to prevent this costly check from running on every page load
*/
if (Yii::app()->request->getParam('checkResponseTableFields') == 'y') {
foreach ($oSurveys as $oSurvey) {
// This actually clears the schema cache, not just refreshes it
$oDB->schema->refresh();
// We get the active surveys
if ($oSurvey->isActive && $oSurvey->hasResponsesTable) {
$model = SurveyDynamic::model($oSurvey->sid);
$aColumns = $model->getMetaData()->columns;
$aQids = array();

// We get the columns of the responses table
foreach ($aColumns as $oColumn) {
// Question columns start with the SID
if (strpos((string) $oColumn->name, (string)$oSurvey->sid) !== false) {
// Fileds are separated by X
$aFields = explode('X', (string) $oColumn->name);

if (isset($aFields[1])) {
$sGid = $aFields[1];

// QID field can be more than just QID, like: 886other or 886A1
// So we clean it by finding the first alphabetical character
$sDirtyQid = $aFields[2];
preg_match('~[a-zA-Z_#]~i', $sDirtyQid, $match, PREG_OFFSET_CAPTURE);

if (isset($match[0][1])) {
$sQID = substr($sDirtyQid, 0, $match[0][1]);
} else {
// It was just the QID.... (maybe)
$sQID = $sDirtyQid;
}

// Here, we get the question as defined in backend
try {
$oQuestion = Question::model()->findByAttributes(['qid' => $sQID , 'sid' => $oSurvey->sid]);
} catch (Exception $e) {
// QID potentially invalid , see #17458, reset $oQuestion
$oQuestion = null;
}
if (is_a($oQuestion, 'Question')) {
// We check if its GID is the same as the one defined in the column name
if ($oQuestion->gid != $sGid) {
// If not, we change the column name
$sNvColName = $oSurvey->sid . 'X' . $oQuestion->group->gid . 'X' . $sDirtyQid;

if (array_key_exists($sNvColName, $aColumns)) {
// This case will not happen often, only when QID + Subquestion ID == QID of a question in the target group
// So we'll change the group of the question question group table (so in admin interface, not in frontend)
$oQuestion->gid = $sGid;
$oQuestion->save();
} else {
$oTransaction = $oDB->beginTransaction();
$oDB->createCommand()->renameColumn($model->tableName(), $oColumn->name, $sNvColName);
$oTransaction->commit();
// Here, we get the question as defined in backend
try {
$oQuestion = Question::model()->findByAttributes(['qid' => $sQID , 'sid' => $oSurvey->sid]);
} catch (Exception $e) {
// QID potentially invalid , see #17458, reset $oQuestion
$oQuestion = null;
}
if (is_a($oQuestion, 'Question')) {
// We check if its GID is the same as the one defined in the column name
if ($oQuestion->gid != $sGid) {
// If not, we change the column name
$sNvColName = $oSurvey->sid . 'X' . $oQuestion->group->gid . 'X' . $sDirtyQid;

if (array_key_exists($sNvColName, $aColumns)) {
// This case will not happen often, only when QID + Subquestion ID == QID of a question in the target group
// So we'll change the group of the question question group table (so in admin interface, not in frontend)
$oQuestion->gid = $sGid;
$oQuestion->save();
} else {
$oTransaction = $oDB->beginTransaction();
$oDB->createCommand()->renameColumn($model->tableName(), $oColumn->name, $sNvColName);
$oTransaction->commit();
}
}
} else {
// QID not found: The function to split the fieldname into the SGQA data is not 100% reliable
// So for certain question types (for example Text Array) the field name cannot be properly derived
// In this case just ignore the field - see also https://bugs.limesurvey.org/view.php?id=15642
// There is still a extremely low chance that an unwanted rename happens if a collision like this happens in the same survey
}
} else {
// QID not found: The function to split the fieldname into the SGQA data is not 100% reliable
// So for certain question types (for example Text Array) the field name cannot be properly derived
// In this case just ignore the field - see also https://bugs.limesurvey.org/view.php?id=15642
// There is still a extremely low chance that an unwanted rename happens if a collision like this happens in the same survey
}
}
}
}
}
}

$oDB->schema->refresh();
$oDB->schemaCachingDuration = 3600;
$oDB->schema->getTables();
$oDB->active = false;
$oDB->active = true;
User::model()->refreshMetaData();
Yii::app()->db->schema->getTable('{{surveys}}', true);
Yii::app()->db->schema->getTable('{{templates}}', true);
Survey::model()->refreshMetaData();
$oDB->schema->refresh();
$oDB->schemaCachingDuration = 3600;
$oDB->schema->getTables();
$oDB->active = false;
$oDB->active = true;
User::model()->refreshMetaData();
Yii::app()->db->schema->getTable('{{surveys}}', true);
Yii::app()->db->schema->getTable('{{templates}}', true);
Survey::model()->refreshMetaData();
}
/* Check method before using #14596 */
if (method_exists(Yii::app()->cache, 'flush')) {
Yii::app()->cache->flush();
Expand Down Expand Up @@ -877,20 +884,19 @@ protected function checkintegrity()
$oCriteria = new CDbCriteria();
$oCriteria->compare('scope', 'T');
$assessments = Assessment::model()->findAll($oCriteria);

$sSurveyIDs = Yii::app()->db->createCommand("select sid from {{surveys}}")->queryColumn();
foreach ($assessments as $assessment) {
$iAssessmentCount = count(Survey::model()->findAllByPk($assessment['sid']));
if (!$iAssessmentCount) {
if (!in_array($assessment['sid'], $sSurveyIDs)) {
$aDelete['assessments'][] = array('id' => $assessment['id'], 'assessment' => $assessment['name'], 'reason' => gT('No matching survey'));
}
}

$oCriteria = new CDbCriteria();
$oCriteria->compare('scope', 'G');
$assessments = Assessment::model()->findAll($oCriteria);
$groupIds = Yii::app()->db->createCommand("select gid from {{groups}}")->queryColumn();
foreach ($assessments as $assessment) {
$iAssessmentCount = count(QuestionGroup::model()->findAllByPk(array('gid' => $assessment['gid'], 'language' => $assessment['language'])));
if (!$iAssessmentCount) {
if (!in_array($assessment['gid'], $groupIds)) {
$aDelete['assessments'][] = array('id' => $assessment['id'], 'assessment' => $assessment['name'], 'reason' => gT('No matching group'));
}
}
Expand Down Expand Up @@ -921,30 +927,27 @@ protected function checkintegrity()
/* Check survey languagesettings and restore them if they don't exist */
/***************************************************************************/

$surveys = Survey::model()->findAll();
$surveyModel = new SurveyLight();
$surveys = $surveyModel->findAll();
foreach ($surveys as $survey) {
$aLanguages = $survey->additionalLanguages;
$aLanguages[] = $survey->language;
$languages = Yii::app()->db->createCommand("select surveyls_language from {{surveys_languagesettings}} where surveyls_survey_id=" . $survey->sid)->queryColumn();
foreach ($aLanguages as $langname) {
if ($langname) {
$oLanguageSettings = SurveyLanguageSetting::model()->find('surveyls_survey_id=:surveyid AND surveyls_language=:langname', array(':surveyid' => $survey->sid, ':langname' => $langname));
// A simple find starts to eat up memory, so we need to free it
gc_collect_cycles();
if (!$oLanguageSettings) {
$oLanguageSettings = new SurveyLanguageSetting();
$languagedetails = getLanguageDetails($langname);
$insertdata = array(
'surveyls_survey_id' => $survey->sid,
'surveyls_language' => $langname,
'surveyls_title' => '',
'surveyls_dateformat' => $languagedetails['dateformat']
);
foreach ($insertdata as $k => $v) {
$oLanguageSettings->$k = $v;
}
$oLanguageSettings->save();
$bDirectlyFixed = true;
if (!in_array($langname, $languages)) {
$oLanguageSettings = new SurveyLanguageSetting();
$languagedetails = getLanguageDetails($langname);
$insertdata = array(
'surveyls_survey_id' => $survey->sid,
'surveyls_language' => $langname,
'surveyls_title' => '',
'surveyls_dateformat' => $languagedetails['dateformat']
);
foreach ($insertdata as $k => $v) {
$oLanguageSettings->$k = $v;
}
$oLanguageSettings->save();
$bDirectlyFixed = true;
}
}
}
Expand Down Expand Up @@ -1039,12 +1042,8 @@ protected function checkintegrity()
$aFullOldSIDs[$iSurveyID][] = $sTable;
}
$aOldSIDs = array_unique($aOldSIDs);
$surveys = Survey::model()->findAll();

$aSIDs = array();
foreach ($surveys as $survey) {
$aSIDs[] = $survey['sid'];
}
$aSIDs = Yii::app()->db->createCommand("select sid from {{surveys}}")->queryColumn();
foreach ($aOldSIDs as $iOldSID) {
if (!in_array($iOldSID, $aSIDs)) {
foreach ($aFullOldSIDs[$iOldSID] as $sTableName) {
Expand Down Expand Up @@ -1108,12 +1107,7 @@ protected function checkintegrity()
$aFullOldTokenSIDs[$iSurveyID][] = $sTable;
}
$aOldTokenSIDs = array_unique($aTokenSIDs);
$surveys = Survey::model()->findAll();

$aSIDs = array();
foreach ($surveys as $survey) {
$aSIDs[] = $survey['sid'];
}
$aSIDs = Yii::app()->db->createCommand("select sid from {{surveys}}")->queryColumn();
foreach ($aOldTokenSIDs as $iOldTokenSID) {
if (!in_array($iOldTokenSID, $aOldTokenSIDs)) {
foreach ($aFullOldTokenSIDs[$iOldTokenSID] as $sTableName) {
Expand Down
2 changes: 1 addition & 1 deletion application/controllers/admin/DataEntry.php
View file
Expand Up @@ -2010,7 +2010,7 @@ private function returnClosedAccessSurveyErrorMessage(): string
private function returnAccessCodeIsNotValidOrAlreadyInUseErrorMessage(): string
{
$errormsg = CHtml::tag('div', array('class' => 'warningheader'), gT("Error"));
$errormsg .= CHtml::tag('p', array(), gT("The access code have provided is not valid or has already been used."));
$errormsg .= CHtml::tag('p', array(), gT("The provided access code is not valid or has already been used."));
return $errormsg;
}

Expand Down
15 changes: 2 additions & 13 deletions application/helpers/common_helper.php
View file
Expand Up @@ -4429,21 +4429,10 @@ function fixSubquestions()
->limit(10000)
->query();
$aRecords = $surveyidresult->readAll();

$dbVersionNumber = SettingGlobal::getDBVersionNumber();

if ($dbVersionNumber < 148) {
$aQuestionTypes = QuestionType::modelsAttributes();
} else {
$aQuestionTypes = QuestionTheme::findQuestionMetaDataForAllTypes(); //be careful!!! only use this if QuestionTheme already exists (see updateDB ...)
}
$aQuestionTypes = QuestionTheme::findQuestionMetaDataForAllTypes(); //be careful!!! only use this if QuestionTheme already exists (see updateDB ...)
while (count($aRecords) > 0) {
foreach ($aRecords as $sv) {
if ($dbVersionNumber < 148) {
$hasSubquestions = $aQuestionTypes[$sv['type']]['subquestions'];
} else {
$hasSubquestions = (int)$aQuestionTypes[$sv['type']]['settings']->subquestions;
}
$hasSubquestions = (int)$aQuestionTypes[$sv['type']]['settings']->subquestions;
if ($hasSubquestions) {
// If the question type allows subquestions, set the type in each subquestion
Yii::app()->db->createCommand("update {{questions}} set type='{$sv['type']}', gid={$sv['gid']} where qid={$sv['qid']}")->execute();
Expand Down
39 changes: 38 additions & 1 deletion application/helpers/update/updates/Update_148.php
View file
Expand Up @@ -88,6 +88,43 @@ public function up()
// Add language field to question_attributes table
addColumn('{{question_attributes}}', 'language', "string(20)");
upgradeQuestionAttributes148();
fixSubquestions();
$this->fixSubquestions148();
}

private function fixSubquestions148()
{
$surveyidresult = $this->db->createCommand()
->select('sq.qid, q.gid , q.type ')
->from('{{questions}} sq')
->join('{{questions}} q', 'sq.parent_qid=q.qid')
->where('sq.parent_qid>0 AND (sq.gid!=q.gid or sq.type!=q.type)')
->limit(10000)
->query();
$aRecords = $surveyidresult->readAll();
$aQuestionTypes = \QuestionType::modelsAttributes();
while (count($aRecords) > 0) {
foreach ($aRecords as $sv) {
$hasSubquestions = $aQuestionTypes[$sv['type']]['subquestions'];
if ($hasSubquestions) {
// If the question type allows subquestions, set the type in each subquestion
$this->db->createCommand("update {{questions}} set type='{$sv['type']}', gid={$sv['gid']} where qid={$sv['qid']}")->execute();
} else {
// If the question type doesn't allow subquestions, delete each subquestion
// Model is used because more tables are involved.
$oSubquestion = \Question::model()->find("qid=:qid", array("qid" => $sv['qid']));
if (!empty($oSubquestion)) {
$oSubquestion->delete();
}
}
}
$surveyidresult = $this->db->createCommand()
->select('sq.qid, q.gid , q.type ')
->from('{{questions}} sq')
->join('{{questions}} q', 'sq.parent_qid=q.qid')
->where('sq.parent_qid>0 AND (sq.gid!=q.gid or sq.type!=q.type)')
->limit(10000)
->query();
$aRecords = $surveyidresult->readAll();
}
}
}

0 comments on commit ad0a1d6

Please sign in to comment.