From f779b4fbc230cef1718674ff07a849433fc286b1 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Fri, 24 Oct 2014 06:34:36 +0200 Subject: [PATCH] improved code, put more into config, fixed some issues such as auto termination, need to move class into files once I know whether we create a new plugin for this --- config/global.ini.php | 15 +- plugins/CoreConsole/Commands/TestRunOnAws.php | 611 ++++++++++++------ 2 files changed, 422 insertions(+), 204 deletions(-) diff --git a/config/global.ini.php b/config/global.ini.php index f085a9a147c..82a6161ec77 100644 --- a/config/global.ini.php +++ b/config/global.ini.php @@ -38,12 +38,15 @@ type = InnoDB schema = Mysql -[aws] -accesskey = "XXXXXX" -secret = "XXXXXXXXXXXXXXXXXX" -keyname = "name" -securitygroups[] = "default" -pem_file = "" +[tests] +aws_accesskey = "" +aws_secret = "" +aws_keyname = "" +aws_securitygroups[] = "default" +aws_pem_file = "" +aws_region = "us-east-1" +aws_ami = "ami-b69c1ade" +aws_instance_type = "c3.large" [log] ; possible values for log: screen, database, file diff --git a/plugins/CoreConsole/Commands/TestRunOnAws.php b/plugins/CoreConsole/Commands/TestRunOnAws.php index c470c5e8e61..7039c0d2a40 100644 --- a/plugins/CoreConsole/Commands/TestRunOnAws.php +++ b/plugins/CoreConsole/Commands/TestRunOnAws.php @@ -14,6 +14,7 @@ use Aws\CloudWatch\Enum\Statistic; use Aws\CloudWatch\Enum\Unit; use Piwik\Config; +use Piwik\Development; use Piwik\Plugin\ConsoleCommand; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -28,6 +29,20 @@ class Ssh extends \Net_SSH2 */ private $output; + public static function connectToAws($host, $pemFile) + { + $key = new \Crypt_RSA(); + $key->loadKey(file_get_contents($pemFile)); + + $ssh = new Ssh($host); + + if (!$ssh->login('ubuntu', $key)) { + throw new \RuntimeException("Login to $host using $pemFile failed"); + } + + return $ssh; + } + public function setOutput(OutputInterface $output) { $this->output = $output; @@ -38,7 +53,7 @@ public function exec($command, $callback = null) $command = 'cd www/piwik && ' . $command; $output = $this->output; - $output->writeln("Executing $command"); + $output->writeln("Executing $command"); return parent::exec($command, function($tempOutput) use ($output) { $output->write($tempOutput); @@ -46,295 +61,495 @@ public function exec($command, $callback = null) } } -/** - */ -class TestRunOnAws extends ConsoleCommand +class AwsConfig { - private $region = 'us-east-1'; - private $ami = 'ami-b69c1ade'; - - protected function configure() + public function getRegion() { - $this->setName('tests:run-aws'); - $this->addArgument('testsuite', InputArgument::OPTIONAL, 'integration, system, all or ui'); - $this->addOption('launch-only', null, InputOption::VALUE_NONE); - $this->addOption('update-only', null, InputOption::VALUE_NONE); + return trim($this->getConfigValue('aws_region')); + } - $gitHash = trim(`git rev-parse HEAD`); - $this->addOption('checkout', null, InputOption::VALUE_REQUIRED, 'Git hash or branch to checkout. Defaults to current hash', $gitHash); - $this->setDescription('Run a specific testsuite on AWS'); + public function getAmi() + { + return trim($this->getConfigValue('aws_ami')); } - /** - * Execute command like: ./console core:clear-caches - */ - protected function execute(InputInterface $input, OutputInterface $output) + public function getInstanceType() { - $testSuite = $this->getTestSuite($input); - $launchOnly = $input->getOption('launch-only'); - $updateOnly = $input->getOption('update-only'); + return trim($this->getConfigValue('aws_instance_type')); + } - // Todo check correct configuration such as pem file exists etc + public function getKeyName() + { + return $this->getConfigValue('aws_keyname'); + } - if (empty($testSuite) && empty($launchOnly) && empty($updateOnly)) { - throw new \InvalidArgumentException('Either provide a testsuite argument or define to launch / update only'); - } + public function getPemFile() + { + return trim($this->getConfigValue('aws_pem_file')); + } - $name = 'PiwikTestSuite-' . $testSuite; + public function getAccessKey() + { + return trim($this->getConfigValue('aws_accesskey')); + } - $instances = $this->launchOrReuseRunningInstance($name); + public function getSecretKey() + { + return trim($this->getConfigValue('aws_secret')); + } - $host = current($instances->getPath('Reservations/*/Instances/*/PublicDnsName')); - $pemFile = $this->getPemFile(); + public function getSecurityGroups() + { + return $this->getConfigValue('aws_securitygroups'); + } - $output->writeln("Access instance using ssh -i $pemFile ubuntu@$host"); - $output->writeln("You can log in to Piwik via root:secure at $host"); - $output->writeln("You can access database via root:secure (mysql -uroot -psecure)"); - $output->writeln("Files are located in ~/www/piwik"); + public function validate() + { + $configKeysToValidate = array( + 'aws_accesskey', + 'aws_secret', + 'aws_region', + 'aws_ami', + 'aws_instance_type', + 'aws_pem_file', + 'aws_keyname', + 'aws_securitygroups', + ); - if ($launchOnly) { - return 0; + foreach ($configKeysToValidate as $key) { + if (!$this->getConfigValue($key)) { + throw new \RuntimeException("[tests]$key is not configured"); + } } - $key = new \Crypt_RSA(); - $key->loadKey(file_get_contents($pemFile)); + $pemFile = $this->getPemFile(); - $ssh = new Ssh($host); - $ssh->setOutput($output); - if (!$ssh->login('ubuntu', $key)) { - $output->writeln('Login Failed'); + if (!file_exists($pemFile)) { + throw new \RuntimeException('[tests]aws_pem_file the file does not exist or is not readable'); } + } - $gitHash = $input->getOption('checkout'); - $this->updatePiwik($ssh, $gitHash, $host); + private function getConfig() + { + return Config::getInstance()->tests; + } - if ($updateOnly) { - $ssh->disconnect(); - $output->writeln("Repository updated and checked out " . $gitHash); + private function getConfigValue($key) + { + $config = $this->getConfig(); - return 0; - } + return $config[$key]; + } +} - $this->runTests($ssh, $testSuite); +class AwsTags +{ + /** + * @var Ec2Client + */ + private $ec2Client; - if (in_array($testSuite, array('system', 'all'))) { - $output->writeln("Tests finished. You can browse processed files at $host/tests/PHPUnit/System/processed/"); - } elseif ('ui' === $testSuite) { - $output->writeln("Tests finished. You can browse processed screenshots at $host/tests/PHPUnit/UI/processed-ui-screenshots/"); - } else { - $output->writeln("Tests finished"); - } + public function __construct(Ec2Client $client) + { + $this->ec2Client = $client; + } - $ssh->disconnect(); + public function assignTagsToInstances($instanceIds, $instanceName) + { + $this->ec2Client->createTags(array( + 'Resources' => $instanceIds, + 'Tags' => array( + array( + 'Key' => 'Name', + 'Value' => $instanceName, + ) + ), + )); } +} - private function updatePiwik(\Net_SSH2 $ssh, $gitHash, $host) +class AwsCloudWatch +{ + /** + * @var AwsConfig + */ + private $config; + + /** + * @var CloudWatchClient + */ + private $client; + + public function __construct(AwsConfig $awsConfig) { - $ssh->exec('git reset --hard'); - $ssh->exec('git clean -d -f'); - $ssh->exec('git fetch --all'); - $ssh->exec('git checkout ' . trim($gitHash)); - $ssh->exec('sudo composer.phar self-update'); - $ssh->exec('composer.phar install'); - $ssh->exec('php --version'); - $ssh->exec('mysql --version'); - $ssh->exec('phantomjs --version'); + $this->config = $awsConfig; + $this->client = CloudWatchClient::factory($this->getConnectionOptions()); + } - $ssh->exec('cp ./tests/PHPUnit/phpunit.xml.dist ./tests/PHPUnit/phpunit.xml'); - $ssh->exec("sed -i 's/@REQUEST_URI@/\\//g' ./tests/PHPUnit/phpunit.xml"); - $ssh->exec("sed -i 's/amazonAwsUrl/$host/g' ./config/config.ini.php"); + public function terminateInstanceIfIdleForTooLong($instanceIds) + { + $this->client->putMetricAlarm(array( + 'AlarmName' => 'TerminateInstanceBecauseIdle', + 'AlarmDescription' => 'Terminate instances if CPU is on average < 10% for 5 minutes in a row 8 times consecutively', + 'ActionsEnabled' => true, + 'OKActions' => array(), + 'AlarmActions' => $this->getAlarmActions(), + 'InsufficientDataActions' => array(), + 'MetricName' => 'CPUUtilization', + 'Namespace' => $this->getNamespace(), + 'Statistic' => Statistic::AVERAGE, + 'Dimensions' => $this->getDimensions($instanceIds), + 'Period' => 300, + 'Unit' => Unit::PERCENT, + 'EvaluationPeriods' => 8, + 'Threshold' => 10, + 'ComparisonOperator' => ComparisonOperator::LESS_THAN_THRESHOLD, + )); + + $this->client->putMetricAlarm(array( + 'AlarmName' => 'TerminateInstanceIfStatusCheckFails', + 'AlarmDescription' => 'Terminate instances in case two status check fail within one minute', + 'ActionsEnabled' => true, + 'OKActions' => array(), + 'AlarmActions' => $this->getAlarmActions(), + 'InsufficientDataActions' => array(), + 'MetricName' => 'StatusCheckFailed', + 'Namespace' => $this->getNamespace(), + 'Statistic' => Statistic::AVERAGE, + 'Dimensions' => $this->getDimensions($instanceIds), + 'Period' => 60, + 'Unit' => Unit::PERCENT, + 'EvaluationPeriods' => 2, + 'Threshold' => 1, + 'ComparisonOperator' => ComparisonOperator::GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + )); } - private function getTestSuite(InputInterface $input) + private function getConnectionOptions() { - $allowed = array('integration', 'system', 'all', 'ui'); + return array( + 'key' => $this->config->getAccessKey(), + 'secret' => $this->config->getSecretKey(), + 'region' => $this->config->getRegion() + ); + } - $testsuite = $input->getArgument('testsuite'); + private function getDimensions($instanceIds) + { + $dimensions = array(); - if (!empty($testsuite) && !in_array($testsuite, $allowed)) { - throw new \InvalidArgumentException('Test suite argument is wrong, use one of following: ' . implode(', ', $allowed)); + foreach ($instanceIds as $instanceId) { + $dimensions[] = array( + 'Name' => 'InstanceId', + 'Value' => $instanceId, + ); } - return $testsuite; + return $dimensions; + } + + private function getNamespace() + { + return 'AWS/EC2'; + } + + private function getAlarmActions() + { + return array( + 'arn:aws:automate:' . $this->config->getRegion() . ':ec2:terminate', + 'arn:aws:sns:' . $this->config->getRegion() . ':682510200394:TerminateInstanceBecauseIdle' + ); + } + +} + +class AwsInstances +{ + + /** + * @var AwsConfig + */ + private $config; + + /** + * @var Ec2Client + */ + private $client; + + public function __construct(AwsConfig $config) + { + $this->config = $config; + $this->client = $this->createEc2Client(); + } + + public function findExisting() + { + $instances = $this->client->describeInstances(array( + 'Filters' => array( + array('Name' => 'image-id', 'Values' => array($this->config->getAmi())), + array('Name' => 'key-name', 'Values' => array($this->config->getKeyName())), + array('Name' => 'instance-state-name', 'Values' => array('running')), + // array('Name' => 'tag:Name', 'Values' => array($name)), uncomment if you want to create an instance per testsuite + ) + )); + + return $instances; + } + + public function terminate($instanceIds) + { + $this->client->terminateInstances(array( + 'InstanceIds' => $instanceIds + )); + + $this->client->waitUntilInstanceTerminated(array( + 'InstanceIds' => $instanceIds + )); } - private function launchInstance(Ec2Client $ec2Client, $name) + public function launch() { - $result = $ec2Client->runInstances(array( - 'Name' => $name, - 'ImageId' => $this->ami, + $result = $this->client->runInstances(array( + 'ImageId' => $this->config->getAmi(), 'MinCount' => 1, 'MaxCount' => 1, - 'InstanceType' => 'c3.large', - 'KeyName' => $this->getKeyName(), - 'SecurityGroups' => $this->getConfigValue('securitygroups'), + 'InstanceType' => $this->config->getInstanceType(), + 'KeyName' => $this->config->getKeyName(), + 'SecurityGroups' => $this->config->getSecurityGroups(), 'InstanceInitiatedShutdownBehavior' => 'terminate' )); $instanceIds = $result->getPath('Instances/*/InstanceId'); - // TODO tag instance - try { - - $ec2Client->waitUntilInstanceRunning(array( - 'InstanceIds' => $instanceIds, - )); - - $instances = $ec2Client->describeInstances(array( - 'InstanceIds' => $instanceIds, - )); + return $instanceIds; + } - $this->terminateInstancesOnceInactive($instanceIds); + public function setup($instanceIds, $instanceName) + { + $this->client->waitUntilInstanceRunning(array( + 'InstanceIds' => $instanceIds, + )); - $ec2Client->createTags(array( - 'Resources' => $instanceIds, - 'Tags' => array( - array( - 'Key' => 'Name', - 'Value' => $name, - ) - ), - )); + $awsCloudWatch = new AwsCloudWatch($this->config); + $awsCloudWatch->terminateInstanceIfIdleForTooLong($instanceIds); - return $instances; + $awsTags = new AwsTags($this->client); + $awsTags->assignTagsToInstances($instanceIds, $instanceName); - } catch (\Exception $e) { - $this->terminateInstances($instanceIds); + $instances = $this->client->describeInstances(array( + 'InstanceIds' => $instanceIds, + )); - throw new \RuntimeException('We failed to launch a new instance so we terminated them directly again. Try again! Error Message: ' . $e->getMessage()); - } + return $instances; } - private function getKeyName() + private function createEc2Client() { - return $this->getConfigValue('keyname'); + return Ec2Client::factory($this->getConnectionOptions()); } - private function getPemFile() + private function getConnectionOptions() { - return $this->getConfigValue('pem_file'); + return array( + 'key' => $this->config->getAccessKey(), + 'secret' => $this->config->getSecretKey(), + 'region' => $this->config->getRegion() + ); } +} - private function getConfig() +class Aws { + + /** + * @var AwsConfig + */ + private $config; + + public function __construct(AwsConfig $config) { - return Config::getInstance()->aws; + $this->config = $config; } - private function getConfigValue($key) + public function launchOrResumeInstance($instanceName) { - $config = $this->getConfig(); + $awsInstances = new AwsInstances($this->config); + $instances = $awsInstances->findExisting(); - return $config[$key]; + $reservations = $instances->getPath('Reservations'); + + if (empty($reservations)) { + $instanceIds = $awsInstances->launch(); + + try { + $instances = $awsInstances->setup($instanceIds, $instanceName); + } catch(\Exception $e) { + $awsInstances->terminate($instanceIds); + + throw new \RuntimeException('We failed to launch a new instance so we terminated it directly. Try again! Error Message: ' . $e->getMessage()); + } + } + + $host = current($instances->getPath('Reservations/*/Instances/*/PublicDnsName')); + + return $host; } - private function launchOrReuseRunningInstance($name) +} + +class RemoteTestRunner +{ + /** + * @var \Net_SSH2 + */ + private $ssh; + + public function construct(\Net_SSH2 $ssh) { - $ec2Client = $this->createEc2Client(); + $this->ssh = $ssh; + } - $instances = $ec2Client->describeInstances(array( - 'DryRun' => false, - 'Filters' => array( - array('Name' => 'image-id', 'Values' => array($this->ami)), - array('Name' => 'key-name', 'Values' => array($this->getKeyName())), - array('Name' => 'instance-state-name', 'Values' => array('running')), - // array('Name' => 'tag:Name', 'Values' => array($name)), - ) - )); + public function updatePiwik($gitHash) + { + $this->ssh->exec('git reset --hard'); + $this->ssh->exec('git clean -d -f'); + $this->ssh->exec('git fetch --all'); + $this->ssh->exec('git checkout ' . trim($gitHash)); + $this->ssh->exec('sudo composer.phar self-update'); + $this->ssh->exec('composer.phar install'); + } - $reservations = $instances->getPath('Reservations'); + public function runTests($host, $testSuite) + { + $this->prepareTestRun($host); + $this->printVersionInfo(); + $this->doRunTests($testSuite); + } - if (empty($reservations)) { - $instances = $this->launchInstance($ec2Client, $name); - } + private function prepareTestRun($host) + { + $this->ssh->exec('cp ./tests/PHPUnit/phpunit.xml.dist ./tests/PHPUnit/phpunit.xml'); + $this->ssh->exec("sed -i 's/@REQUEST_URI@/\\//g' ./tests/PHPUnit/phpunit.xml"); + $this->ssh->exec("sed -i 's/amazonAwsUrl/$host/g' ./config/config.ini.php"); + } - return $instances; + private function printVersionInfo() + { + $this->ssh->exec('php --version'); + $this->ssh->exec('mysql --version'); + $this->ssh->exec('phantomjs --version'); } - private function runTests(\Net_SSH2 $ssh, $testSuite) + private function doRunTests($testSuite) { if ('all' === $testSuite) { - $ssh->exec('php console tests:run --options="--colors"'); + $this->ssh->exec('php console tests:run --options="--colors"'); } elseif ('ui' === $testSuite) { - $ssh->exec('php console tests:run-ui'); + $this->ssh->exec('php console tests:run-ui'); } else { - $ssh->exec('php console tests:run --options="--colors" --testsuite="' . $testSuite . '"'); + $this->ssh->exec('php console tests:run --options="--colors" --testsuite="unit"'); + $this->ssh->exec('php console tests:run --options="--colors" --testsuite="' . $testSuite . '"'); } } +} - private function getConnectionOptions() +/** + * To enable this command you have to enable development mode + */ +class TestRunOnAws extends ConsoleCommand +{ + private $allowedTestSuites = array('integration', 'system', 'all', 'ui'); + + public function isEnabled() { - return array( - 'key' => $this->getConfigValue('accesskey'), - 'secret' => $this->getConfigValue('secret'), - 'region' => $this->region - ); + return Development::isEnabled(); } - private function terminateInstancesOnceInactive($instanceIds) + protected function configure() { - $dimensions = array(); - foreach ($instanceIds as $instanceId) { - $dimensions[] = array( - 'Name' => 'instance-id', - 'Value' => $instanceId, - ); + $this->setName('tests:run-aws'); + $this->addArgument('testsuite', InputArgument::OPTIONAL, 'Allowed values: ' . implode(', ', $this->allowedTestSuites)); + $this->addOption('launch-only', null, InputOption::VALUE_NONE); + $this->addOption('update-only', null, InputOption::VALUE_NONE); + $this->addOption('checkout', null, InputOption::VALUE_REQUIRED, 'Git hash, tag or branch to checkout. Defaults to current hash', $this->getCurrentGitHash()); + $this->setDescription('Run a specific testsuite on AWS'); + } + + /** + * Execute command like: ./console core:clear-caches + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + $testSuite = $this->getTestSuite($input); + $launchOnly = $input->getOption('launch-only'); + $updateOnly = $input->getOption('update-only'); + + if (empty($testSuite) && empty($launchOnly) && empty($updateOnly)) { + throw new \InvalidArgumentException('Either provide a testsuite argument or define --launch-only or --update-only'); } - $alarmActions = array( - 'arn:aws:automate:' . $this->region . ':ec2:terminate', - 'arn:aws:sns:' . $this->region . ':682510200394:TerminateInstanceBecauseIdle' - ); + $name = 'PiwikTestSuite-' . $testSuite; - $namespace = 'AWS/EC2'; + $awsConfig = new AwsConfig(); + $awsConfig->validate(); - $watchClient = CloudWatchClient::factory($this->getConnectionOptions()); - $watchClient->putMetricAlarm(array( - 'AlarmName' => 'TerminateInstanceBecauseIdle', - 'AlarmDescription' => 'Terminate instances if CPU is on average < 10% for 5 minutes in a row 8 times consecutively', - 'ActionsEnabled' => true, - 'OKActions' => array(), - 'AlarmActions' => $alarmActions, - 'InsufficientDataActions' => array(), - 'MetricName' => 'CPUUtilization', - 'Namespace' => $namespace, - 'Statistic' => Statistic::AVERAGE, - 'Dimensions' => $dimensions, - 'Period' => 300, - 'Unit' => Unit::PERCENT, - 'EvaluationPeriods' => 8, - 'Threshold' => 10, - 'ComparisonOperator' => ComparisonOperator::LESS_THAN_THRESHOLD, - )); - $watchClient->putMetricAlarm(array( - 'AlarmName' => 'TerminateInstanceIfStatusCheckFails', - 'AlarmDescription' => 'Terminate instances in case two status check fail within one minute', - 'ActionsEnabled' => true, - 'OKActions' => array(), - 'AlarmActions' => $alarmActions, - 'InsufficientDataActions' => array(), - 'MetricName' => 'StatusCheckFailed', - 'Namespace' => $namespace, - 'Statistic' => Statistic::AVERAGE, - 'Dimensions' => $dimensions, - 'Period' => 60, - 'Unit' => Unit::PERCENT, - 'EvaluationPeriods' => 2, - 'Threshold' => 1, - 'ComparisonOperator' => ComparisonOperator::GREATER_THAN_OR_EQUAL_TO_THRESHOLD, - )); + $aws = new Aws($awsConfig); + $host = $aws->launchOrResumeInstance($name); + + $pemFile = $awsConfig->getPemFile(); + + $output->writeln("Access instance using ssh -i $pemFile ubuntu@$host"); + $output->writeln("You can log in to Piwik via root:secure at http://$host"); + $output->writeln("You can access database via root:secure (mysql -uroot -psecure)"); + $output->writeln("Files are located in ~/www/piwik"); + $output->writeln(' '); + + if ($launchOnly) { + return 0; + } + + $ssh = Ssh::connectToAws($host, $pemFile); + $ssh->setOutput($output); + + $gitHash = $input->getOption('checkout'); + + $testRunner = new RemoteTestRunner($ssh); + $testRunner->updatePiwik($gitHash); + + if ($updateOnly) { + $ssh->disconnect(); + $output->writeln("Repository updated and checked out " . $gitHash); + + return 0; + } + + $testRunner->runTests($host, $testSuite); + + if (in_array($testSuite, array('system', 'all'))) { + $output->writeln("Tests finished. You can browse processed files at http://$host/tests/PHPUnit/System/processed/"); + } elseif ('ui' === $testSuite) { + $output->writeln("Tests finished. You can browse processed screenshots at http://$host/tests/PHPUnit/UI/processed-ui-screenshots/"); + } else { + $output->writeln("Tests finished"); + } + + $ssh->disconnect(); } - private function terminateInstances($instanceIds) + private function getTestSuite(InputInterface $input) { - $ec2Client = $this->createEc2Client(); - $ec2Client->terminateInstances(array( - 'InstanceIds' => $instanceIds - )); + $testsuite = $input->getArgument('testsuite'); + + if (!empty($testsuite) && !in_array($testsuite, $this->allowedTestSuites)) { + throw new \InvalidArgumentException('Test suite argument is wrong, use one of following: ' . implode(', ', $this->allowedTestSuites)); + } + + return $testsuite; } - private function createEc2Client() + private function getCurrentGitHash() { - return Ec2Client::factory($this->getConnectionOptions()); + return trim(`git rev-parse HEAD`); } + }