Fixed issue #17409: Exporting responses with big data sets and activa…

…ted encryption will timeout php memory limit, even with high settings (512M)
LimeSurvey · Aug 13, 2021 · b72beac · Shnoulle · Oct 13, 2022 · b72beac
1 parent bf72986
commit b72beac
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 13 deletions.
diff --git a/application/helpers/admin/export/SurveyDao.php b/application/helpers/admin/export/SurveyDao.php
@@ -81,9 +81,9 @@ public function loadSurveyById($id, $lang = null, FormattingOptions $oOptions =
      * @param string $completionState all, complete or incomplete
      * @param array $aFields If empty all, otherwise only select the selected fields from the survey response table
      * @param string $sResponsesId
-     * @throws CException
+     * @return CDbCommand
      */
-    public function loadSurveyResults(SurveyObj $survey, $iMinimum, $iMaximum, $sFilter = '', $completionState = 'all', $aFields = array(), $sResponsesId = null)
+    public function loadSurveyResults(SurveyObj $survey, $iMinimum, $iMaximum, $sFilter = '', $completionState = 'all', $aFields = array(), $sResponsesId = null): CDbCommand
     {
         $oSurvey = Survey::model()->findByPk($survey->id);
 
@@ -107,7 +107,7 @@ public function loadSurveyResults(SurveyObj $survey, $iMinimum, $iMaximum, $sFil
             foreach ($aTokenFields as &$sField) {
                 $sField = "tokentable." . $sField;
             }
-            $aSelectFields = array_merge($aSelectFields, array_diff($aTokenFields, array('tokentable.token')));
+            $aSelectFields = array_merge($aSelectFields, array_diff($aTokenFields, ['tokentable.token']));
             //$aSelectFields=array_diff($aSelectFields, array('{{survey_{$survey->id}}}.token'));
             //$aSelectFields[]='{{survey_' . $survey->id . '}}.token';
         }
@@ -117,29 +117,29 @@ public function loadSurveyResults(SurveyObj $survey, $iMinimum, $iMaximum, $sFil
             foreach ($aTimingFields as &$sField) {
                 $sField = "survey_timings." . $sField;
             }
-            $aSelectFields = array_merge($aSelectFields, array_diff($aTimingFields, array('survey_timings.id')));
+            $aSelectFields = array_merge($aSelectFields, array_diff($aTimingFields, ['survey_timings.id']));
             //$aSelectFields=array_diff($aSelectFields, array('{{survey_{$survey->id}}}.id'));
             //$aSelectFields[]='{{survey_' . $survey->id . '}}.id';
         }
         if (empty($sResponsesId)) {
-            $aParams = array(
+            $aParams = [
                 'min' => $iMinimum,
                 'max' => $iMaximum
-            );
+            ];
             $selection = $oSurvey->responsesTableName . '.id >= :min AND ' . $oSurvey->responsesTableName . '.id <= :max';
             $oRecordSet->where($selection, $aParams);
         } else {
             $aResponsesId = explode(',', $sResponsesId);
 
             foreach ($aResponsesId as $i => $iResponseId) {
-                $iResponseId = (int) $iResponseId;
+                $iResponseId = (int)$iResponseId;
                 $selection = $oSurvey->responsesTableName . '.id = :id' . $i;
 
 
                 if ($i === 0) {
-                    $oRecordSet->where($selection, array('id' . $i => $iResponseId));
+                    $oRecordSet->where($selection, ['id' . $i => $iResponseId]);
                 } else {
-                    $oRecordSet->orWhere($selection, array('id' . $i => $iResponseId));
+                    $oRecordSet->orWhere($selection, ['id' . $i => $iResponseId]);
                 }
             }
         }
@@ -165,6 +165,7 @@ public function loadSurveyResults(SurveyObj $survey, $iMinimum, $iMaximum, $sFil
                 break;
         }
         $oRecordSet->order = $oSurvey->responsesTableName . '.id ASC';
-        $survey->responses = $oRecordSet->select($aSelectFields)->query();
+        $oRecordSet->select($aSelectFields);
+        return $oRecordSet;
     }
 }
diff --git a/application/helpers/admin/exportresults_helper.php b/application/helpers/admin/exportresults_helper.php
@@ -103,9 +103,20 @@ function exportResponses($iSurveyId, $sLanguageCode, $sExportPlugin, FormattingO
         $surveyDao = new SurveyDao();
         $survey = $surveyDao->loadSurveyById($iSurveyId, $sLanguageCode, $oOptions);
         $writer->init($survey, $sLanguageCode, $oOptions);
-
-        $surveyDao->loadSurveyResults($survey, $oOptions->responseMinRecord, $oOptions->responseMaxRecord, $sFilter, $oOptions->responseCompletionState, $oOptions->selectedColumns, $oOptions->aResponses);
-        $writer->write($survey, $sLanguageCode, $oOptions, true);
+
+        $countResponsesCommand = $surveyDao->loadSurveyResults($survey, $oOptions->responseMinRecord, $oOptions->responseMaxRecord, $sFilter, $oOptions->responseCompletionState, $oOptions->selectedColumns, $oOptions->aResponses);
+        $countResponsesCommand->select('count(*)');
+        $reponsesCount = $countResponsesCommand->queryScalar();
+        $maxRows = 100;
+        $maxPages = ceil($reponsesCount / $maxRows);
+        for ($i = 0; $i < $maxPages; $i++) {
+            $offset = $i * $maxRows;
+            $responsesQuery = $surveyDao->loadSurveyResults($survey, $oOptions->responseMinRecord, $oOptions->responseMaxRecord, $sFilter, $oOptions->responseCompletionState, $oOptions->selectedColumns, $oOptions->aResponses);
+            $responsesQuery->offset($offset);
+            $responsesQuery->limit($maxRows);
+            $survey->responses = $responsesQuery->query();
+            $writer->write($survey, $sLanguageCode, $oOptions, true);
+        }
         $result = $writer->close();
 
         // Close resultset if needed