From 29d87bea0185db6051a8f72a8714062f3fd1436e Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Mon, 17 Sep 2018 18:56:01 +0530 Subject: [PATCH 01/14] Add placeholder migration Signed-off-by: Kirtan Gajjar --- migrations/1_blank.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 migrations/1_blank.php diff --git a/migrations/1_blank.php b/migrations/1_blank.php new file mode 100644 index 000000000..aafb96fa8 --- /dev/null +++ b/migrations/1_blank.php @@ -0,0 +1,16 @@ + Date: Mon, 17 Sep 2018 18:56:21 +0530 Subject: [PATCH 02/14] Change cache directory path Signed-off-by: Kirtan Gajjar --- php/class-ee.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php/class-ee.php b/php/class-ee.php index 472af447b..ae1353a0c 100644 --- a/php/class-ee.php +++ b/php/class-ee.php @@ -94,7 +94,7 @@ public static function get_cache() { if ( ! $cache ) { $home = Utils\get_home_dir(); - $dir = getenv( 'EE_CACHE_DIR' ) ? : "$home/.ee/cache"; + $dir = getenv( 'EE_CACHE_DIR' ) ? : '/opt/easyengine/.cache/'; // 6 months, 300mb $cache = new FileCache( $dir, 15552000, 314572800 ); From 0579330dde5d5f63f2c8067cbb12a63dc068fb6a Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Mon, 17 Sep 2018 18:57:00 +0530 Subject: [PATCH 03/14] Invalidate cache before update don't migrate after put in /tmp Signed-off-by: Kirtan Gajjar --- php/commands/src/CLI_Command.php | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/php/commands/src/CLI_Command.php b/php/commands/src/CLI_Command.php index 815d2af1a..eb02033c7 100644 --- a/php/commands/src/CLI_Command.php +++ b/php/commands/src/CLI_Command.php @@ -1,9 +1,9 @@ migrate(); - } - } - - /** - * Function to run migrations required to upgrade to the newer version. Will always be invoked from the newer phar downloaded inside the /tmp folder - */ - private function migrate() { - $rsp = new \EE\RevertableStepProcessor(); - $rsp->add_step( 'ee-migrations', 'EE\Migration\Executor::execute_migrations' ); - $rsp->add_step( 'site-migration', 'EE\Migration\Containers::start_container_migration' ); - $rsp->execute(); } /** @@ -319,6 +305,10 @@ public function update( $_, $assoc_args ) { } else { EE::error( "md5 hash for download ({$md5_file}) is different than the release hash ({$release_hash})." ); } + + $cache = EE::get_cache(); + $cache->write( 'migrated', '0' ); + $php_binary = Utils\get_php_binary(); $process = EE\Process::create( "{$php_binary} $temp cli info" ); $result = $process->run(); From ec07dc87226985e17bbf0bed80b57e715b07115a Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Mon, 17 Sep 2018 18:57:17 +0530 Subject: [PATCH 04/14] Fix migrations Signed-off-by: Kirtan Gajjar --- php/EE/Migration/Executor.php | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/php/EE/Migration/Executor.php b/php/EE/Migration/Executor.php index bcc0ebfef..3587b0c40 100644 --- a/php/EE/Migration/Executor.php +++ b/php/EE/Migration/Executor.php @@ -2,14 +2,13 @@ namespace EE\Migration; -use \EE; -use \EE\Model\Migration; -use \EE\Utils; -use Symfony\Component\Finder\Finder; +use EE; +use EE\Model\Migration; +use EE\Utils; class Executor { - const MIGRATION_PATH = EE_ROOT . '/migrations'; + const MIGRATION_PATH = EE_ROOT . '/migrations/'; /** * Executes all pending migrations @@ -17,13 +16,12 @@ class Executor { public static function execute_migrations() { Utils\delem_log( "ee migration start" ); - EE::log( "Migrating EasyEngine data to new version" ); + EE::log( "Executing migrations" ); $migrations = self::get_migrations_to_execute(); if( empty( $migrations ) ) { EE::success( "Noting to migrate" ); - exit( 0 ); } sort( $migrations ); @@ -103,13 +101,18 @@ private static function get_migrations_to_execute() { } private static function get_migrations_from_db() { - return Migration::all(); + return array_column( Migration::all(), 'migration' ); } private static function get_migrations_from_fs() { // array_slice is used to remove . and .. returned by scandir() - $migrations = array_slice( scandir( self::MIGRATION_PATH ), 2 ); - array_walk( $migrations, function( &$migration, $index ) { + $migrations = scandir( self::MIGRATION_PATH ); + + if( ! Utils\inside_phar() ) { + $migrations = array_slice( scandir( self::MIGRATION_PATH ), 2 ); + } + + array_walk( $migrations, function( &$migration, $index ) { $migration = rtrim( $migration, '.php' ); }); return $migrations; From f8e79e28e92a4d944ecd8230ba17d63b2bc86611 Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Mon, 17 Sep 2018 18:58:09 +0530 Subject: [PATCH 05/14] Add cached call to verify migration have ran Signed-off-by: Kirtan Gajjar --- php/EE/Runner.php | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/php/EE/Runner.php b/php/EE/Runner.php index 248f042f5..9f4bf4d53 100644 --- a/php/EE/Runner.php +++ b/php/EE/Runner.php @@ -3,12 +3,12 @@ namespace EE; use EE; -use EE\Utils; use EE\Dispatcher; use EE\Dispatcher\CompositeCommand; -use Mustangostang\Spyc; +use EE\Utils; use Monolog\Logger; -use EE\Model\Site; +use Mustangostang\Spyc; + /** * Performs the execution of a command. * @@ -59,6 +59,24 @@ private function init_ee() { define( 'WEBROOT', \EE\Utils\trailingslashit( $this->config['sites_path'] ) ); define( 'DB', EE_CONF_ROOT.'/ee.sqlite' ); define( 'LOCALHOST_IP', '127.0.0.1' ); + + $cache = EE::get_cache(); + + if ( ! $cache->has( 'migrated' ) || '1' !== $cache->read( 'migrated' ) ) { + if ( ! $this->migrate() ) { + EE::error( 'There was some error while migrating.' ); + } + $cache->write( 'migrated', '1' ); + } + } + + /** + * Function to run migrations required to upgrade to the newer version. Will always be invoked from the newer phar downloaded inside the /tmp folder + */ + private function migrate() { + $rsp = new \EE\RevertableStepProcessor(); + $rsp->add_step( 'ee-migrations', 'EE\Migration\Executor::execute_migrations' ); + return $rsp->execute(); } /** From 490d6927ad764524b25b7e092c2f36e8589ab6b7 Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Mon, 17 Sep 2018 18:58:25 +0530 Subject: [PATCH 06/14] Correct make phar for migration Signed-off-by: Kirtan Gajjar --- utils/make-phar.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/utils/make-phar.php b/utils/make-phar.php index 7e4ef7d91..866c46ab1 100644 --- a/utils/make-phar.php +++ b/utils/make-phar.php @@ -16,9 +16,9 @@ require EE_VENDOR_DIR . '/autoload.php'; require EE_ROOT . '/php/utils.php'; -use Symfony\Component\Finder\Finder; -use EE\Utils; use EE\Configurator; +use EE\Utils; +use Symfony\Component\Finder\Finder; $configurator = new Configurator( EE_ROOT . '/utils/make-phar-spec.php' ); @@ -161,6 +161,7 @@ function get_composer_versions( $current_version ) { ->ignoreVCS(true) ->name('*.php') ->in(EE_ROOT . '/php') + ->in(EE_ROOT . '/migrations') ->in(EE_VENDOR_DIR . '/mustache') ->in(EE_VENDOR_DIR . '/rmccue/requests') ->in(EE_VENDOR_DIR . '/composer') @@ -221,6 +222,15 @@ function get_composer_versions( $current_version ) { add_file( $phar, $file ); } +if( 2 === count( scandir(__DIR__ . '/../migrations') ) ) { + $finder = new Finder(); + $finder->directories()->name('migrations')->in(EE_ROOT); + + foreach ( $finder as $file ) { + add_file( $phar, $file ); + } +} + $finder = new Finder(); $finder ->files() From c9b211ba5259c099276239ebe73ebd82c2e49f15 Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Tue, 18 Sep 2018 14:15:52 +0530 Subject: [PATCH 07/14] Add option table and migration Signed-off-by: Kirtan Gajjar --- php/EE/Model/Option.php | 53 +++++++++++++++++++++++++++++++++++++++++ php/class-ee-db.php | 6 +++++ 2 files changed, 59 insertions(+) create mode 100644 php/EE/Model/Option.php diff --git a/php/EE/Model/Option.php b/php/EE/Model/Option.php new file mode 100644 index 000000000..f026b0a87 --- /dev/null +++ b/php/EE/Model/Option.php @@ -0,0 +1,53 @@ + $key, + 'value' => $value, + ] + ); + } + + $existing_key->value = $value; + + return $existing_key->save(); + } + + /** + * Gets a key from options table + * + * @param string $key Key of option + * + * @throws \Exception + * @return bool|Option + */ + public function get( string $key ) { + $option = static::find( $key ); + + return false === $option ? false : $option->value; + } +} diff --git a/php/class-ee-db.php b/php/class-ee-db.php index 0533eb362..6461fb301 100644 --- a/php/class-ee-db.php +++ b/php/class-ee-db.php @@ -127,6 +127,12 @@ private static function create_required_tables() { PRIMARY KEY (id) );'; + $query .= 'CREATE TABLE options ( + key VARCHAR NOT NULL, + value VARCHAR NOT NULL, + PRIMARY KEY (key) + );'; + try { self::$pdo->exec( $query ); } catch ( PDOException $exception ) { From 2ce1a036e76540623ac39a4db091f4a422a27f10 Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Tue, 18 Sep 2018 14:16:18 +0530 Subject: [PATCH 08/14] Update migration trigger logic Signed-off-by: Kirtan Gajjar --- php/EE/Runner.php | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/php/EE/Runner.php b/php/EE/Runner.php index 9f4bf4d53..f69057989 100644 --- a/php/EE/Runner.php +++ b/php/EE/Runner.php @@ -2,9 +2,11 @@ namespace EE; +use Composer\Semver\Comparator; use EE; use EE\Dispatcher; use EE\Dispatcher\CompositeCommand; +use EE\Model\Option; use EE\Utils; use Monolog\Logger; use Mustangostang\Spyc; @@ -60,14 +62,7 @@ private function init_ee() { define( 'DB', EE_CONF_ROOT.'/ee.sqlite' ); define( 'LOCALHOST_IP', '127.0.0.1' ); - $cache = EE::get_cache(); - - if ( ! $cache->has( 'migrated' ) || '1' !== $cache->read( 'migrated' ) ) { - if ( ! $this->migrate() ) { - EE::error( 'There was some error while migrating.' ); - } - $cache->write( 'migrated', '1' ); - } + $this->maybe_trigger_migration(); } /** @@ -796,6 +791,23 @@ private function auto_check_update() { exit; } + /** + * Triggers migration if current phar version > version in ee_option table + */ + private function maybe_trigger_migration() { + $db_version = Option::get( 'version' ); + $current_version = preg_replace( '/-nightly$/', '', EE_VERSION ); + + if ( Comparator::lessThan( $current_version, $db_version ) ) { + EE::error( 'It seems you\'re not running latest version. Please download and run latest version of EasyEngine' ); + } elseif ( Comparator::greaterThan( $current_version, $db_version ) ) { + if ( ! $this->migrate() ) { + EE::error( 'There was some error while migrating. Please check logs.' ); + } + Option::set( 'version', $current_version ); + } + } + /** * Get a suggestion on similar (sub)commands when the user entered an * unknown (sub)command. From 66fb18e834c8f15a945976461a77a84a153ef609 Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Wed, 19 Sep 2018 13:10:27 +0530 Subject: [PATCH 09/14] Removed cache write call Signed-off-by: Kirtan Gajjar --- php/commands/src/CLI_Command.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/php/commands/src/CLI_Command.php b/php/commands/src/CLI_Command.php index eb02033c7..dc41e8613 100644 --- a/php/commands/src/CLI_Command.php +++ b/php/commands/src/CLI_Command.php @@ -306,9 +306,6 @@ public function update( $_, $assoc_args ) { EE::error( "md5 hash for download ({$md5_file}) is different than the release hash ({$release_hash})." ); } - $cache = EE::get_cache(); - $cache->write( 'migrated', '0' ); - $php_binary = Utils\get_php_binary(); $process = EE\Process::create( "{$php_binary} $temp cli info" ); $result = $process->run(); From de53431fb8f00733a02b6720eb22b78d395dd62f Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Wed, 19 Sep 2018 13:10:56 +0530 Subject: [PATCH 10/14] Remove repetetive scandir call Signed-off-by: Kirtan Gajjar --- php/EE/Migration/Executor.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/php/EE/Migration/Executor.php b/php/EE/Migration/Executor.php index 3587b0c40..560226826 100644 --- a/php/EE/Migration/Executor.php +++ b/php/EE/Migration/Executor.php @@ -22,6 +22,7 @@ public static function execute_migrations() { if( empty( $migrations ) ) { EE::success( "Noting to migrate" ); + return; } sort( $migrations ); @@ -105,11 +106,11 @@ private static function get_migrations_from_db() { } private static function get_migrations_from_fs() { - // array_slice is used to remove . and .. returned by scandir() $migrations = scandir( self::MIGRATION_PATH ); if( ! Utils\inside_phar() ) { - $migrations = array_slice( scandir( self::MIGRATION_PATH ), 2 ); + // array_slice is used to remove . and .. returned by scandir() + $migrations = array_slice( $migrations, 2 ); } array_walk( $migrations, function( &$migration, $index ) { From c1adcfaa0ef9942f975b69cb9eb31bd36bf0823d Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Wed, 19 Sep 2018 13:11:14 +0530 Subject: [PATCH 11/14] Align = sign Signed-off-by: Kirtan Gajjar --- php/EE/Runner.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php/EE/Runner.php b/php/EE/Runner.php index f69057989..5cf21ee9d 100644 --- a/php/EE/Runner.php +++ b/php/EE/Runner.php @@ -795,7 +795,7 @@ private function auto_check_update() { * Triggers migration if current phar version > version in ee_option table */ private function maybe_trigger_migration() { - $db_version = Option::get( 'version' ); + $db_version = Option::get( 'version' ); $current_version = preg_replace( '/-nightly$/', '', EE_VERSION ); if ( Comparator::lessThan( $current_version, $db_version ) ) { From dc0af0bcb5b163f415f348e44041fc6a7740e2ed Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Wed, 19 Sep 2018 13:18:56 +0530 Subject: [PATCH 12/14] Fix typo Signed-off-by: Kirtan Gajjar --- php/EE/Migration/Executor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php/EE/Migration/Executor.php b/php/EE/Migration/Executor.php index 560226826..f4e33d8f1 100644 --- a/php/EE/Migration/Executor.php +++ b/php/EE/Migration/Executor.php @@ -21,7 +21,7 @@ public static function execute_migrations() { $migrations = self::get_migrations_to_execute(); if( empty( $migrations ) ) { - EE::success( "Noting to migrate" ); + EE::success( "Nothing to migrate" ); return; } From 2e40c68eccf065df29ded08888e15b6af9064414 Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Wed, 19 Sep 2018 13:25:31 +0530 Subject: [PATCH 13/14] Renamed changed log to debug Signed-off-by: Kirtan Gajjar --- php/EE/Migration/Executor.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/php/EE/Migration/Executor.php b/php/EE/Migration/Executor.php index f4e33d8f1..78c7e82f8 100644 --- a/php/EE/Migration/Executor.php +++ b/php/EE/Migration/Executor.php @@ -16,12 +16,12 @@ class Executor { public static function execute_migrations() { Utils\delem_log( "ee migration start" ); - EE::log( "Executing migrations" ); + EE::debug( "Executing migrations" ); $migrations = self::get_migrations_to_execute(); if( empty( $migrations ) ) { - EE::success( "Nothing to migrate" ); + EE::debug( "Nothing to migrate" ); return; } @@ -34,7 +34,7 @@ public static function execute_migrations() { exit( 1 ); } - EE::success( "Successfully migrated EasyEngine" ); + EE::debug( "Successfully migrated EasyEngine" ); } /** @@ -68,7 +68,7 @@ private static function execute_migration_stack( $migrations ) { } try { - EE::log( "Migrating: $migrations[0]" ); + EE::debug( "Migrating: $migrations[0]" ); $migration->up(); Migration::create([ @@ -77,7 +77,7 @@ private static function execute_migration_stack( $migrations ) { ]); $migration->status = 'complete'; - EE::log( "Migrated: $migrations[0]" ); + EE::debug( "Migrated: $migrations[0]" ); $remaining_migrations = array_splice( $migrations, 1, count( $migrations ) ); self::execute_migration_stack( $remaining_migrations ); } @@ -85,9 +85,9 @@ private static function execute_migration_stack( $migrations ) { if( $migration->status !== 'complete' ) { EE::error( "Errors were encountered while processing: $migrations[0]\n" . $e->getMessage(), false ); } - EE::log( "Reverting: $migrations[0]" ); + EE::debug( "Reverting: $migrations[0]" ); $migration->down(); - EE::log( "Reverted: $migrations[0]" ); + EE::debug( "Reverted: $migrations[0]" ); throw $e; } } From 425ff14dcaaa7f60668a3f9129e6b7b6c7461881 Mon Sep 17 00:00:00 2001 From: Kirtan Gajjar Date: Wed, 19 Sep 2018 13:59:52 +0530 Subject: [PATCH 14/14] Added log message before updating or migrating Signed-off-by: Kirtan Gajjar --- php/EE/Runner.php | 20 +++++++++++++++----- php/commands/src/CLI_Command.php | 2 ++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/php/EE/Runner.php b/php/EE/Runner.php index 5cf21ee9d..4d381c2c5 100644 --- a/php/EE/Runner.php +++ b/php/EE/Runner.php @@ -796,16 +796,26 @@ private function auto_check_update() { */ private function maybe_trigger_migration() { $db_version = Option::get( 'version' ); - $current_version = preg_replace( '/-nightly$/', '', EE_VERSION ); + $current_version = preg_replace( '/-nightly.*$/', '', EE_VERSION ); + + if ( ! $db_version ) { + $this->trigger_migration( $current_version ); + return; + } if ( Comparator::lessThan( $current_version, $db_version ) ) { EE::error( 'It seems you\'re not running latest version. Please download and run latest version of EasyEngine' ); } elseif ( Comparator::greaterThan( $current_version, $db_version ) ) { - if ( ! $this->migrate() ) { - EE::error( 'There was some error while migrating. Please check logs.' ); - } - Option::set( 'version', $current_version ); + EE::log( 'Executing migrations. This might take some time.' ); + $this->trigger_migration( $current_version ); + } + } + + private function trigger_migration( $version ) { + if ( ! $this->migrate() ) { + EE::error( 'There was some error while migrating. Please check logs.' ); } + Option::set( 'version', $version ); } /** diff --git a/php/commands/src/CLI_Command.php b/php/commands/src/CLI_Command.php index dc41e8613..5254a633d 100644 --- a/php/commands/src/CLI_Command.php +++ b/php/commands/src/CLI_Command.php @@ -306,6 +306,8 @@ public function update( $_, $assoc_args ) { EE::error( "md5 hash for download ({$md5_file}) is different than the release hash ({$release_hash})." ); } + EE::log( 'Updating EasyEngine to new version. This might take some time.' ); + $php_binary = Utils\get_php_binary(); $process = EE\Process::create( "{$php_binary} $temp cli info" ); $result = $process->run();