Change "fetch translations" task to fetch them in batches.

The request limit was in place already, but the client didn't respect it before, leading to unhappy users in case they had lots of 3rd party modules installed, or more than 2-3 locales enabled. This is all taken care of now.

modules/gallery/helpers/gallery_task.php

@@ -122,14 +122,15 @@ static function update_l10n(&$task) {
       $start = microtime(true);
       $data = Cache::instance()->get("update_l10n_cache:{$task->id}");
       if ($data) {
-        list($dirs, $files, $cache) = unserialize($data);
+        list($dirs, $files, $cache, $num_fetched) = unserialize($data);
       }
       $i = 0;
 
       switch ($task->get("mode", "init")) {
       case "init":  // 0%
         $dirs = array("gallery", "modules", "themes", "installer");
         $files = $cache = array();
+        $num_fetched = 0;
         $task->set("mode", "find_files");
         $task->status = t("Finding files");
         break;
@@ -161,7 +162,7 @@ static function update_l10n(&$task) {
         }
         break;
 
-      case "scan_files": // 10% - 90%
+      case "scan_files": // 10% - 70%
         while (($file = array_pop($files)) && microtime(true) - $start < 0.5) {
           $file = DOCROOT . $file;
           switch (pathinfo($file, PATHINFO_EXTENSION)) {
@@ -179,25 +180,31 @@ static function update_l10n(&$task) {
         $task->status = t2("Scanning files: scanned 1 file",
                            "Scanning files: scanned %count files", $total_files - count($files));
 
-        $task->percent_complete = 10 + 80 * ($total_files - count($files)) / $total_files;
+        $task->percent_complete = 10 + 60 * ($total_files - count($files)) / $total_files;
         if (empty($files)) {
           $task->set("mode", "fetch_updates");
           $task->status = t("Fetching updates");
-          $task->percent_complete = 90;
+          $task->percent_complete = 70;
         }
         break;
 
-      case "fetch_updates":  // 90% - 100%
-        l10n_client::fetch_updates();
-        $task->done = true;
-        $task->state = "success";
-        $task->status = t("Translations installed/updated");
-        $task->percent_complete = 100;
+      case "fetch_updates":  // 70% - 100%
+        // Send fetch requests in batches until we're done
+        $num_remaining = l10n_client::fetch_updates($num_fetched);
+        if ($num_remaining) {
+          $total = $num_fetched + $num_remaining;
+          $task->percent_complete = 70 + 30 * ((float) $num_fetched / $total);
+        } else {
+          $task->done = true;
+          $task->state = "success";
+          $task->status = t("Translations installed/updated");
+          $task->percent_complete = 100;
+        }
       }
 
-      if ($task->percent_complete < 100) {
+      if (!$task->done) {
         Cache::instance()->set("update_l10n_cache:{$task->id}",
-                               serialize(array($dirs, $files, $cache)));
+                               serialize(array($dirs, $files, $cache, $num_fetched)));
       } else {
         Cache::instance()->delete("update_l10n_cache:{$task->id}");
       }

modules/gallery/helpers/l10n_client.php

@@ -68,9 +68,15 @@ static function validate_api_key($api_key) {
   }
 
   /**
-   * @return an array of messages that will be written to the task log
+   * Fetches translations for l10n messages. Must be called repeatedly
+   * until 0 is returned (which is a countdown indicating progress).
+   *
+   * @param $num_fetched in/out parameter to specify which batch of
+   *     messages to fetch translations for.
+   * @return The number of messages for which we didn't fetch
+   *     translations for.
    */
-  static function fetch_updates() {
+  static function fetch_updates(&$num_fetched) {
     $request->locales = array();
     $request->messages = new stdClass();
 
@@ -79,31 +85,50 @@ static function fetch_updates() {
       $request->locales[] = $locale;
     }
 
-    // @todo Batch requests (max request size)
-    foreach (db::build()
-             ->select("key", "locale", "revision", "translation")
-             ->from("incoming_translations")
-             ->execute() as $row) {
+    // See the server side code for how we arrive at this
+    // number as a good limit for #locales * #messages.
+    $max_messages = 2000 / count($locales);
+    $num_messages = 0;
+    $rows = db::build()
+        ->select("key", "locale", "revision", "translation")
+        ->from("incoming_translations")
+        ->order_by("key")
+        ->limit(1000000)  // ignore, just there to satisfy SQL syntax
+        ->offset($num_fetched)
+        ->execute();
+    $num_remaining = $rows->count();
+    foreach ($rows as $row) {
       if (!isset($request->messages->{$row->key})) {
+        if ($num_messages >= $max_messages) {
+          break;
+        }
         $request->messages->{$row->key} = 1;
+        $num_messages++;
       }
-      if (!empty($row->revision) && !empty($row->translation)) {
+      if (!empty($row->revision) && !empty($row->translation) &&
+          isset($locales[$row->locale])) {
         if (!is_object($request->messages->{$row->key})) {
           $request->messages->{$row->key} = new stdClass();
         }
-        $request->messages->{$row->key}->{$row->locale} = $row->revision;
+        $request->messages->{$row->key}->{$row->locale} = (int) $row->revision;
       }
+      $num_fetched++;
+      $num_remaining--;
     }
     // @todo Include messages from outgoing_translations?
 
+    if (!$num_messages) {
+      return $num_remaining;
+    }
+
     $request_data = json_encode($request);
     $url = self::_server_url() . "?q=translations/fetch";
     list ($response_data, $response_status) = remote::post($url, array("data" => $request_data));
     if (!remote::success($response_status)) {
       throw new Exception("@todo TRANSLATIONS_FETCH_REQUEST_FAILED " . $response_status);
     }
     if (empty($response_data)) {
-      return array(t("Translations fetch request resulted in an empty response"));
+      return $num_remaining;
     }
 
     $response = json_decode($response_data);
@@ -150,6 +175,8 @@ static function fetch_updates() {
       $entry->translation = $translation;
       $entry->save();
     }
+
+    return $num_remaining;
   }
 
   static function submit_translations() {