Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
  • 2 commits
  • 3 files changed
  • 0 comments
  • 1 contributor
2  lib/democracyinaction-php
... ... @@ -1 +1 @@
1   -Subproject commit 4b680765cfbccb9246e0a5c3d174c7037dedee40
  1 +Subproject commit b6e304afad003df46a6cd2b3d79497be3ec6f437
45 scripts/dispatcher.js
@@ -4,31 +4,38 @@ var spawn = require('child_process').spawn;
4 4 var exec = require('child_process').exec;
5 5 var migrations = require('./migrations');
6 6
  7 +var table_migrations = [];
7 8 _.each(migrations, function(migration){
8   - var migration_functions = [];
9   - exec('php fetch_table_data.php '+migration.table, function(err, table_info_json){
10   - var table_info = JSON.parse(table_info_json);
11   - var blocksize = Math.ceil(table_info['table_size'] / migration.concurrency);
12   - _.each(_.range(migration.concurrency), function(i){
13   - migration_functions.push(function(done){
14   - console.log('starting '+migration.table+':'+String(i*blocksize));
  9 + table_migrations.push(function(done){
  10 + var migration_functions = [];
  11 + exec('php fetch_table_data.php '+migration.table, function(err, table_info_json){
  12 + var table_info = JSON.parse(table_info_json);
  13 + var blocksize = Math.ceil(table_info['table_size'] / migration.concurrency);
  14 + _.each(_.range(migration.concurrency), function(i){
  15 + migration_functions.push(function(done){
  16 + console.log('starting '+migration.table+':'+String(i*blocksize));
15 17
16   - var migrate = spawn('php', ['migrate.php', migration.table, migration.model, table_info['special_array'], String(i*blocksize), String(blocksize)]);
  18 + var migrate = spawn('php', ['migrate.php', migration.table, migration.model, table_info['special_array'], String(i*blocksize), String(blocksize)]);
17 19
18   - migrate.on('exit', function(code){
19   - console.log('exited '+migration.table+':'+String(i*blocksize)+'\n');
20   - if(code == 1) return done(1);
21   - done(null);
22   - });
  20 + migrate.on('exit', function(code){
  21 + console.log('exited '+migration.table+':'+String(i*blocksize)+'\n');
  22 + if(code == 1) return done(1);
  23 + done(null);
  24 + });
23 25
24   - migrate.stdout.on('data', function(data){
25   - console.log('stdout for '+migration.table+':'+String(i*blocksize)+'\n'+data);
  26 + migrate.stdout.on('data', function(data){
  27 + console.log('stdout for '+migration.table+':'+String(i*blocksize)+'\n'+data);
  28 + });
26 29 });
27 30 });
28   - });
29   - async.parallel(migration_functions, function(err, results){
30   - if(err) return console.log(err);
31   - console.log('Migration of table "'+migration.table+'" complete.');
  31 + async.parallel(migration_functions, function(err, results){
  32 + if(err) return console.log(err);
  33 + console.log('Migration of table "'+migration.table+'" complete.');
  34 + done();
  35 + });
32 36 });
33 37 });
34 38 });
  39 +async.series(table_migrations, function(){
  40 + console.log('All table migrations complete.');
  41 +});
40 scripts/migrate.php
@@ -3,18 +3,38 @@
3 3
4 4 list($cmd, $table, $model, $table_data, $offset, $size) = $argv;
5 5 $table_data = unserialize($table_data);
  6 +$limit = min(500,$size);
6 7 for($x = $offset; $x < $offset + $size; $x += 500){
7   - $limit = min(500,$size);
8   - echo "get: $table, offset: $x, limit: $limit\n";
9   - $results = $dia->get($table,array('limit' => $x.','.$limit));
10   - echo "offset: $offset fetched ".count($results)." results";
11   - foreach($results AS $result){
12   - foreach($result AS $field_key => $field_value){
13   - if(in_array($field_key, $table_data['integer'])) $result[$field_key] = (integer) $field_value;
14   - if(in_array($field_key, $table_data['boolean'])) $result[$field_value] = (bool) $field_value;
  8 + migrate_slice($x, $limit);
  9 +}
  10 +
  11 +function migrate_slice($slice_offset, $limit){
  12 + global $table, $model, $dia, $table_data;
  13 + echo "get: $table, offset: $slice_offset, limit: $limit\n";
  14 + $results = $dia->get($table,array('limit' => $slice_offset.','.$limit));
  15 + try{
  16 + if(gettype($results) == "object" && $results instanceof Exception) throw $results;
  17 + echo "offset: $slice_offset limit: $limit fetched ".count($results)." results";
  18 + foreach($results AS $result){
  19 + foreach($result AS $field_key => $field_value){
  20 + if(in_array($field_key, $table_data['integer'])) $result[$field_key] = (integer) $field_value;
  21 + if(in_array($field_key, $table_data['boolean'])) $result[$field_value] = (bool) $field_value;
  22 + }
  23 + $m = new $model($result);
  24 + $m->db_insert();
  25 + }
  26 + } catch(Exception $e){
  27 + // in case of error, recursively fetch all slices around the error slice
  28 + echo "offset: $slice_offset limit: $limit error!\n";
  29 + echo $e->getMessage()."\n";
  30 + if($limit > 1){
  31 + $slice_1_limit = ceil($limit / 2);
  32 + $slice_2_limit = $limit - $slice_1_limit;
  33 + $slice_1_offset = $slice_offset;
  34 + $slice_2_offset = $slice_offset + $slice_1_limit;
  35 + migrate_slice($slice_1_offset, $slice_1_limit);
  36 + migrate_slice($slice_2_offset, $slice_2_limit);
15 37 }
16   - $m = new $model($result);
17   - $m->db_insert();
18 38 }
19 39 }
20 40 ?>

No commit comments for this range

Something went wrong with that request. Please try again.