Permalink
Browse files

add batching of bulk writes 1000 at a time

  • Loading branch information...
1 parent 2f1f0bb commit 60dd436ecfe75a79cbdd85df4d85ab3ac2a0d5ca @BigBlueHat committed Nov 12, 2011
Showing with 12 additions and 2 deletions.
  1. +12 −2 sql2couchdb.php
View
@@ -209,10 +209,20 @@ function map($row, $json, $mysqlQuery)
$docsToSend = array();
-while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC))
+$c = 0;
+while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC)) {
$docsToSend[] = map($row, clone $jsonFile->doc, $mysqlQuery);
+ $c++;
+ if ($c === 1001) {
+ $sag->bulk($docsToSend);
+ $docsToSend = array();
+ $c = 0;
+ }
+}
-$sag->bulk($docsToSend);
+if (count($docsToSend) > 0) {
+ $sag->bulk($docsToSend);
+}
mysql_free_result($mysqlQuery);
mysql_close($mysqlCon);

1 comment on commit 60dd436

@sbisbee

Yeah, nice. I've been meaning to build something like this into Sag's utils package.

Please sign in to comment.