From 034c5ef096a38946f6844a31aa6422b42413f0d3 Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Mon, 29 Apr 2019 21:51:35 -0400 Subject: [PATCH 1/5] feat(config): Add config file migration-snapshot.php. --- config/migration-snapshot.php | 15 +++++++++++++++ src/Handlers/MigrateFinishedHandler.php | 5 ++--- src/Handlers/MigrateStartingHandler.php | 5 ++--- src/ServiceProvider.php | 10 +++++++++- tests/MigrateHookTest.php | 2 +- 5 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 config/migration-snapshot.php diff --git a/config/migration-snapshot.php b/config/migration-snapshot.php new file mode 100644 index 0000000..91775b5 --- /dev/null +++ b/config/migration-snapshot.php @@ -0,0 +1,15 @@ + env('MIGRATION_SNAPSHOT_ENVIRONMENTS', 'development,local,testing'), +]; diff --git a/src/Handlers/MigrateFinishedHandler.php b/src/Handlers/MigrateFinishedHandler.php index 9062666..3b2fc1c 100644 --- a/src/Handlers/MigrateFinishedHandler.php +++ b/src/Handlers/MigrateFinishedHandler.php @@ -14,8 +14,7 @@ public function handle(CommandFinished $event) 'migrate' === $event->command // CONSIDER: Also `migrate:fresh`. && ! $event->input->hasParameterOption(['--help', '--pretend', '-V', '--version']) && env('MIGRATION_SNAPSHOT', true) - // CONSIDER: Making configurable blacklist of environments. - && 'production' !== app()->environment() + && in_array(app()->environment(), explode(',', config('migration-snapshot.environments')), true) ) { $options = MigrateStartingHandler::inputToArtisanOptions($event->input); $database = $options['--database'] ?? env('DB_CONNECTION'); @@ -28,4 +27,4 @@ public function handle(CommandFinished $event) \Artisan::call('migrate:dump', $options, $event->output); } } -} \ No newline at end of file +} diff --git a/src/Handlers/MigrateStartingHandler.php b/src/Handlers/MigrateStartingHandler.php index 8482475..18aecf8 100644 --- a/src/Handlers/MigrateStartingHandler.php +++ b/src/Handlers/MigrateStartingHandler.php @@ -58,8 +58,7 @@ public function handle(CommandStarting $event) && env('MIGRATION_SNAPSHOT', true) // CONSIDER: Config option. // Never implicitly load fresh (from file) in production since it // would need to drop first, and that would be destructive. - // CONSIDER: Making configurable blacklist of environments. - && 'production' !== app()->environment() + && in_array(app()->environment(), explode(',', config('migration-snapshot.environments')), true) // No point in implicitly loading when it's not present. && file_exists(database_path() . MigrateDumpCommand::SCHEMA_SQL_PATH_SUFFIX) ) { @@ -149,4 +148,4 @@ private static function inputValidateWorkaround($input) : bool return true; } -} \ No newline at end of file +} diff --git a/src/ServiceProvider.php b/src/ServiceProvider.php index 3717aaa..801befa 100644 --- a/src/ServiceProvider.php +++ b/src/ServiceProvider.php @@ -10,6 +10,10 @@ class ServiceProvider extends \Illuminate\Support\ServiceProvider public function boot() { if ($this->app->runningInConsole()) { + $this->publishes([ + __DIR__ . '/../config/migration-snapshot.php' => config_path('migration-snapshot.php'), + ], 'config'); + $this->commands([ MigrateDumpCommand::class, MigrateLoadCommand::class, @@ -19,6 +23,10 @@ public function boot() public function register() { + $this->mergeConfigFrom(__DIR__ . '/../config/migration-snapshot.php', 'migration-snapshot'); + $this->app->register(EventServiceProvider::class); + + parent::register(); } -} \ No newline at end of file +} diff --git a/tests/MigrateHookTest.php b/tests/MigrateHookTest.php index 66de916..364ae01 100644 --- a/tests/MigrateHookTest.php +++ b/tests/MigrateHookTest.php @@ -46,4 +46,4 @@ public function test_handle_doesNotLoadWhenDbHasMigrated() $this->assertEquals(1, \DB::table('test_ms')->count()); } -} \ No newline at end of file +} From b3eea699cd59f5297c1c9e88b36c8be376206c4a Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Mon, 29 Apr 2019 23:05:07 -0400 Subject: [PATCH 2/5] feat(MigrateDumpCommand): Option to reorder `migrations` rows for consistency. --- config/migration-snapshot.php | 19 ++++++++ src/Commands/MigrateDumpCommand.php | 71 +++++++++++++++++++++++------ src/ServiceProvider.php | 2 - tests/Sqlite/MigrateLoadTest.php | 2 +- 4 files changed, 77 insertions(+), 17 deletions(-) diff --git a/config/migration-snapshot.php b/config/migration-snapshot.php index 91775b5..4104d33 100644 --- a/config/migration-snapshot.php +++ b/config/migration-snapshot.php @@ -12,4 +12,23 @@ */ 'environments' => env('MIGRATION_SNAPSHOT_ENVIRONMENTS', 'development,local,testing'), + + /* + |-------------------------------------------------------------------------- + | Whether to reorder the `migrations` rows for consistency. + |-------------------------------------------------------------------------- + | + | The order migrations are applied in development may vary from person to + | person, especially as they are created in parallel. This option reorders + | the migration records for consistency so the output file can be managed + | in source control. + | + | If the order migrations are applied will produce significant differences, + | such as changing the behavior of the app, then this should be left + | disabled. In such cases `migrate:fresh --database=test` followed by + | `migrate` or `migrate:dump` can achieve similar consistency. + | + */ + + 'reorder' => env('MIGRATION_SNAPSHOT_REORDER', false), ]; diff --git a/src/Commands/MigrateDumpCommand.php b/src/Commands/MigrateDumpCommand.php index 861ee50..6f4951d 100644 --- a/src/Commands/MigrateDumpCommand.php +++ b/src/Commands/MigrateDumpCommand.php @@ -56,6 +56,16 @@ public function handle() $this->info('Dumped schema'); } + private static function reorderMigrationRows(array &$output) + { + if (config('migration-snapshot.reorder')) { + sort($output); + foreach ($output as &$line) { + $line = preg_replace('/,\s*[0-9]+\s*\)\s*;\s*$/iu', ',0);', $line); + } + } + } + /** * @param array $db_config like ['host' => , 'port' => ]. * @param string $schema_sql_path like '.../schema.sql' @@ -101,13 +111,22 @@ private static function mysqlDump(array $db_config, string $schema_sql_path) : i } // Include migration rows to avoid unnecessary reruns conflicting. - // CONSIDER: How this could be done as consistent snapshot with - // dump of structure, and avoid duplicate "SET" comments. - passthru( - $command_prefix . ' migrations --no-create-info --skip-extended-insert >> ' - . escapeshellarg($schema_sql_path), + exec( + $command_prefix . ' migrations --no-create-info --skip-extended-insert --compact', + $output, $exit_code ); + if (0 !== $exit_code) { + return $exit_code; + } + + self::reorderMigrationRows($output); + + file_put_contents( + $schema_sql_path, + implode(PHP_EOL, $output), + FILE_APPEND + ); return $exit_code; } @@ -130,7 +149,6 @@ private static function pgsqlDump(array $db_config, string $schema_sql_path) : i . ' --port=' . escapeshellarg($db_config['port']) . ' --username=' . escapeshellarg($db_config['username']) . ' ' . escapeshellarg($db_config['database']); - // TODO: Suppress warning about insecure password. passthru( $command_prefix . ' --file=' . escapeshellarg($schema_sql_path) @@ -142,13 +160,22 @@ private static function pgsqlDump(array $db_config, string $schema_sql_path) : i } // Include migration rows to avoid unnecessary reruns conflicting. - // CONSIDER: How this could be done as consistent snapshot with - // dump of structure, and avoid duplicate "SET" comments. - passthru( - $command_prefix . ' --table=migrations --data-only --inserts >> ' - . escapeshellarg($schema_sql_path), + exec( + $command_prefix . ' --table=migrations --data-only --inserts', + $output, $exit_code ); + if (0 !== $exit_code) { + return $exit_code; + } + + self::reorderMigrationRows($output); + + file_put_contents( + $schema_sql_path, + implode(PHP_EOL, $output), + FILE_APPEND + ); return $exit_code; } @@ -167,24 +194,40 @@ private static function sqliteDump(array $db_config, string $schema_sql_path) : // Since Sqlite lacks Information Schema, and dumping everything may be // too slow or memory intense, just query tables and dump them // individually. + // CONSIDER: Using Laravel's `Schema` code instead. exec($command_prefix . ' .tables', $output, $exit_code); if (0 !== $exit_code) { return $exit_code; } $tables = preg_split('/\s+/', implode(' ', $output)); + file_put_contents($schema_sql_path, ''); + foreach ($tables as $table) { // Only migrations should dump data with schema. $sql_command = 'migrations' === $table ? '.dump' : '.schema'; - passthru( - $command_prefix . ' ' . escapeshellarg("$sql_command $table") - . ' >> ' . escapeshellarg($schema_sql_path), + $output = []; + exec( + $command_prefix . ' ' . escapeshellarg("$sql_command $table"), + $output, $exit_code ); if (0 !== $exit_code) { return $exit_code; } + + if ('migrations' === $table) { + $insert_rows = array_slice($output, 4, -1); + self::reorderMigrationRows($insert_rows); + array_splice($output, 4, -1, $insert_rows); + } + + file_put_contents( + $schema_sql_path, + implode(PHP_EOL, $output) . PHP_EOL, + FILE_APPEND + ); } return $exit_code; diff --git a/src/ServiceProvider.php b/src/ServiceProvider.php index 801befa..65d339e 100644 --- a/src/ServiceProvider.php +++ b/src/ServiceProvider.php @@ -26,7 +26,5 @@ public function register() $this->mergeConfigFrom(__DIR__ . '/../config/migration-snapshot.php', 'migration-snapshot'); $this->app->register(EventServiceProvider::class); - - parent::register(); } } diff --git a/tests/Sqlite/MigrateLoadTest.php b/tests/Sqlite/MigrateLoadTest.php index d8509b4..7288f8d 100644 --- a/tests/Sqlite/MigrateLoadTest.php +++ b/tests/Sqlite/MigrateLoadTest.php @@ -23,4 +23,4 @@ public function test_handle() $this->assertNull(\DB::table('test_ms')->value('name')); } -} \ No newline at end of file +} From e14e19416d9ff6f6d4ef7e2ad6a282b71a580cfa Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Mon, 29 Apr 2019 23:36:08 -0400 Subject: [PATCH 3/5] fix(MigrateDumpCommand): Make reordering numeric and also reorder id column values. --- src/Commands/MigrateDumpCommand.php | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/Commands/MigrateDumpCommand.php b/src/Commands/MigrateDumpCommand.php index 6f4951d..0dbd4a3 100644 --- a/src/Commands/MigrateDumpCommand.php +++ b/src/Commands/MigrateDumpCommand.php @@ -56,14 +56,27 @@ public function handle() $this->info('Dumped schema'); } - private static function reorderMigrationRows(array &$output) + private static function reorderMigrationRows(array $output) : array { if (config('migration-snapshot.reorder')) { - sort($output); - foreach ($output as &$line) { - $line = preg_replace('/,\s*[0-9]+\s*\)\s*;\s*$/iu', ',0);', $line); + $reordered = []; + $new_id = 1; + foreach ($output as $line) { + $occurrences = preg_match( + '/^(.*?VALUES\s*)\([0-9]+,(.*?),\s*[0-9]+\s*\)\s*;\s*$/iu', + $line, + $m + ); + if (1 !== $occurrences) { + throw new \UnexpectedValueException('Only insert rows supported'); + } + $reordered[$m[2]] = "$m[1]($new_id,$m[2],0);"; + $new_id += 1; } + return $reordered; } + + return $output; } /** @@ -120,7 +133,7 @@ private static function mysqlDump(array $db_config, string $schema_sql_path) : i return $exit_code; } - self::reorderMigrationRows($output); + $output = self::reorderMigrationRows($output); file_put_contents( $schema_sql_path, @@ -169,7 +182,7 @@ private static function pgsqlDump(array $db_config, string $schema_sql_path) : i return $exit_code; } - self::reorderMigrationRows($output); + $output = self::reorderMigrationRows($output); file_put_contents( $schema_sql_path, @@ -219,8 +232,8 @@ private static function sqliteDump(array $db_config, string $schema_sql_path) : if ('migrations' === $table) { $insert_rows = array_slice($output, 4, -1); - self::reorderMigrationRows($insert_rows); - array_splice($output, 4, -1, $insert_rows); + $sorted = self::reorderMigrationRows($insert_rows); + array_splice($output, 4, -1, $sorted); } file_put_contents( From 497db85f57a8fee9c94a24e2c73170e0aed6e905 Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Tue, 30 Apr 2019 09:27:56 -0400 Subject: [PATCH 4/5] fix(MigrateDumpCommand): Use only timestamp when sorting migration rows since version string could include problematic characters. --- src/Commands/MigrateDumpCommand.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Commands/MigrateDumpCommand.php b/src/Commands/MigrateDumpCommand.php index 0dbd4a3..63af770 100644 --- a/src/Commands/MigrateDumpCommand.php +++ b/src/Commands/MigrateDumpCommand.php @@ -63,14 +63,16 @@ private static function reorderMigrationRows(array $output) : array $new_id = 1; foreach ($output as $line) { $occurrences = preg_match( - '/^(.*?VALUES\s*)\([0-9]+,(.*?),\s*[0-9]+\s*\)\s*;\s*$/iu', + "/^(.*?VALUES\s*)\([0-9]+,'([0-9_]{17})(.*?),\s*[0-9]+\s*\)\s*;\s*$/iu", $line, $m ); if (1 !== $occurrences) { - throw new \UnexpectedValueException('Only insert rows supported'); + throw new \UnexpectedValueException( + 'Only insert rows supported:' . PHP_EOL . var_export($line, 1) + ); } - $reordered[$m[2]] = "$m[1]($new_id,$m[2],0);"; + $reordered[$m[2]] = "$m[1]($new_id,'$m[2]$m[3],0);"; $new_id += 1; } return $reordered; From 029c28276bedc0f01251934e50edfd650a607dac Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Tue, 30 Apr 2019 09:40:12 -0400 Subject: [PATCH 5/5] chore(MigrateDumpCommand): Tolerate whitespace between comma and quote of migration rows. --- src/Commands/MigrateDumpCommand.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Commands/MigrateDumpCommand.php b/src/Commands/MigrateDumpCommand.php index 63af770..d45f45b 100644 --- a/src/Commands/MigrateDumpCommand.php +++ b/src/Commands/MigrateDumpCommand.php @@ -62,8 +62,10 @@ private static function reorderMigrationRows(array $output) : array $reordered = []; $new_id = 1; foreach ($output as $line) { + // Extract parts of "INSERT ... VALUES ([id],'[ver]',[batch]) + // where version begins with "YYYY_MM_DD_HHMMSS". $occurrences = preg_match( - "/^(.*?VALUES\s*)\([0-9]+,'([0-9_]{17})(.*?),\s*[0-9]+\s*\)\s*;\s*$/iu", + "/^(.*?VALUES\s*)\([0-9]+,\s*'([0-9_]{17})(.*?),\s*[0-9]+\s*\)\s*;\s*$/iu", $line, $m ); @@ -72,6 +74,8 @@ private static function reorderMigrationRows(array $output) : array 'Only insert rows supported:' . PHP_EOL . var_export($line, 1) ); } + // Reassemble parts with new values and index by timestamp of + // version string to sort. $reordered[$m[2]] = "$m[1]($new_id,'$m[2]$m[3],0);"; $new_id += 1; }