Skip to content

Commit

Permalink
MDL-66327 pgsql DML: Update get_records_sql to consume less memory
Browse files Browse the repository at this point in the history
It's more memory efficient to use `pg_fetch_assoc` for each row than to
call `pg_fetch_all` and release memory immediately. This is because we
can treat the assoc fetch like an iterator and it only fetches the
current record into memory one at a time, whilst the all fetch fetches
all records and never unsets them. Attempting to unset them is extremely
time consuming.
  • Loading branch information
andrewnicols committed Aug 8, 2019
1 parent 414eca8 commit 66fa73d
Showing 1 changed file with 12 additions and 18 deletions.
30 changes: 12 additions & 18 deletions lib/dml/pgsql_native_moodle_database.php
Expand Up @@ -842,8 +842,7 @@ public function close_cursor($cursorname) {
* @return array of objects, or empty array if no records were found
* @throws dml_exception A DML specific exception is thrown for any errors.
*/
public function get_records_sql($sql, array $params=null, $limitfrom=0, $limitnum=0) {

public function get_records_sql($sql, array $params = null, $limitfrom = 0, $limitnum = 0) {
list($limitfrom, $limitnum) = $this->normalise_limit_from_num($limitfrom, $limitnum);

if ($limitnum) {
Expand All @@ -868,24 +867,19 @@ public function get_records_sql($sql, array $params=null, $limitfrom=0, $limitnu
}
}

$rows = pg_fetch_all($result);
pg_free_result($result);

$return = array();
if ($rows) {
foreach ($rows as $row) {
$id = reset($row);
if ($blobs) {
foreach ($blobs as $blob) {
$row[$blob] = ($row[$blob] !== null ? pg_unescape_bytea($row[$blob]) : null);
}
}
if (isset($return[$id])) {
$colname = key($row);
debugging("Did you remember to make the first column something unique in your call to get_records? Duplicate value '$id' found in column '$colname'.", DEBUG_DEVELOPER);
$return = [];
while ($row = pg_fetch_assoc($result)) {
$id = reset($row);
if ($blobs) {
foreach ($blobs as $blob) {
$row[$blob] = ($row[$blob] !== null ? pg_unescape_bytea($row[$blob]) : null);
}
$return[$id] = (object)$row;
}
if (isset($return[$id])) {
$colname = key($row);
debugging("Did you remember to make the first column something unique in your call to get_records? Duplicate value '$id' found in column '$colname'.", DEBUG_DEVELOPER);
}
$return[$id] = (object) $row;
}

return $return;
Expand Down

0 comments on commit 66fa73d

Please sign in to comment.