Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
base fork: radicaldesigns/monsterpants
base: 52c91944cb
...
head fork: radicaldesigns/monsterpants
compare: 36c287f104
  • 2 commits
  • 3 files changed
  • 0 commit comments
  • 1 contributor
2  lib/democracyinaction-php
@@ -1 +1 @@
-Subproject commit 4b680765cfbccb9246e0a5c3d174c7037dedee40
+Subproject commit b6e304afad003df46a6cd2b3d79497be3ec6f437
View
45 scripts/dispatcher.js
@@ -4,31 +4,38 @@ var spawn = require('child_process').spawn;
var exec = require('child_process').exec;
var migrations = require('./migrations');
+var table_migrations = [];
_.each(migrations, function(migration){
- var migration_functions = [];
- exec('php fetch_table_data.php '+migration.table, function(err, table_info_json){
- var table_info = JSON.parse(table_info_json);
- var blocksize = Math.ceil(table_info['table_size'] / migration.concurrency);
- _.each(_.range(migration.concurrency), function(i){
- migration_functions.push(function(done){
- console.log('starting '+migration.table+':'+String(i*blocksize));
+ table_migrations.push(function(done){
+ var migration_functions = [];
+ exec('php fetch_table_data.php '+migration.table, function(err, table_info_json){
+ var table_info = JSON.parse(table_info_json);
+ var blocksize = Math.ceil(table_info['table_size'] / migration.concurrency);
+ _.each(_.range(migration.concurrency), function(i){
+ migration_functions.push(function(done){
+ console.log('starting '+migration.table+':'+String(i*blocksize));
- var migrate = spawn('php', ['migrate.php', migration.table, migration.model, table_info['special_array'], String(i*blocksize), String(blocksize)]);
+ var migrate = spawn('php', ['migrate.php', migration.table, migration.model, table_info['special_array'], String(i*blocksize), String(blocksize)]);
- migrate.on('exit', function(code){
- console.log('exited '+migration.table+':'+String(i*blocksize)+'\n');
- if(code == 1) return done(1);
- done(null);
- });
+ migrate.on('exit', function(code){
+ console.log('exited '+migration.table+':'+String(i*blocksize)+'\n');
+ if(code == 1) return done(1);
+ done(null);
+ });
- migrate.stdout.on('data', function(data){
- console.log('stdout for '+migration.table+':'+String(i*blocksize)+'\n'+data);
+ migrate.stdout.on('data', function(data){
+ console.log('stdout for '+migration.table+':'+String(i*blocksize)+'\n'+data);
+ });
});
});
- });
- async.parallel(migration_functions, function(err, results){
- if(err) return console.log(err);
- console.log('Migration of table "'+migration.table+'" complete.');
+ async.parallel(migration_functions, function(err, results){
+ if(err) return console.log(err);
+ console.log('Migration of table "'+migration.table+'" complete.');
+ done();
+ });
});
});
});
+async.series(table_migrations, function(){
+ console.log('All table migrations complete.');
+});
View
40 scripts/migrate.php
@@ -3,18 +3,38 @@
list($cmd, $table, $model, $table_data, $offset, $size) = $argv;
$table_data = unserialize($table_data);
+$limit = min(500,$size);
for($x = $offset; $x < $offset + $size; $x += 500){
- $limit = min(500,$size);
- echo "get: $table, offset: $x, limit: $limit\n";
- $results = $dia->get($table,array('limit' => $x.','.$limit));
- echo "offset: $offset fetched ".count($results)." results";
- foreach($results AS $result){
- foreach($result AS $field_key => $field_value){
- if(in_array($field_key, $table_data['integer'])) $result[$field_key] = (integer) $field_value;
- if(in_array($field_key, $table_data['boolean'])) $result[$field_value] = (bool) $field_value;
+ migrate_slice($x, $limit);
+}
+
+function migrate_slice($slice_offset, $limit){
+ global $table, $model, $dia, $table_data;
+ echo "get: $table, offset: $slice_offset, limit: $limit\n";
+ $results = $dia->get($table,array('limit' => $slice_offset.','.$limit));
+ try{
+ if(gettype($results) == "object" && $results instanceof Exception) throw $results;
+ echo "offset: $slice_offset limit: $limit fetched ".count($results)." results";
+ foreach($results AS $result){
+ foreach($result AS $field_key => $field_value){
+ if(in_array($field_key, $table_data['integer'])) $result[$field_key] = (integer) $field_value;
+ if(in_array($field_key, $table_data['boolean'])) $result[$field_value] = (bool) $field_value;
+ }
+ $m = new $model($result);
+ $m->db_insert();
+ }
+ } catch(Exception $e){
+ // in case of error, recursively fetch all slices around the error slice
+ echo "offset: $slice_offset limit: $limit error!\n";
+ echo $e->getMessage()."\n";
+ if($limit > 1){
+ $slice_1_limit = ceil($limit / 2);
+ $slice_2_limit = $limit - $slice_1_limit;
+ $slice_1_offset = $slice_offset;
+ $slice_2_offset = $slice_offset + $slice_1_limit;
+ migrate_slice($slice_1_offset, $slice_1_limit);
+ migrate_slice($slice_2_offset, $slice_2_limit);
}
- $m = new $model($result);
- $m->db_insert();
}
}
?>

No commit comments for this range

Something went wrong with that request. Please try again.