DEV-706: Worked on Download and Import feature from AWS to local Mysql

config/commands.yaml

@@ -21,9 +21,13 @@ commands:
     class: \Driver\Engines\S3\Upload
   shutdown-sandbox:
     class: \Driver\Engines\MySql\Sandbox\Shutdown
+  download-data-from-s3:
+    class: \Driver\Engines\S3\Download
   call-post-webhook:
     class: \Driver\Commands\Webhook\PostCommand
   setup-environment:
     class: \Driver\Commands\Environment\Setup
   run-transformations:
-    class: \Driver\Engines\MySql\Transformation
+    class: \Driver\Engines\MySql\Transformation
+  import-data-from-system-primary:
+    class: \Driver\Engines\MySql\Import\Primary

config/pipelines.yaml

@@ -39,15 +39,33 @@ pipelines:
       actions:
         - name: run-transformations
           sort: 100
+
+    - name: export-upload-data
+      sort: 500
+      actions:
+        - name: connect
+          sort: 100
         - name: export-data-from-sandbox
           sort: 200
         - name: upload-data-to-s3
           sort: 300
 
     - name: shutdown
-      sort: 500
+      sort: 600
       actions:
         - name: shutdown-sandbox
           sort: 100
 
+    - name: export-s3-db-on-local
+      sort: 700
+      actions:
+        - name: download-data-from-s3
+          sort: 100
+
+    - name: import-local
+      sort: 800
+      actions:
+        - name: import-data-from-system-primary
+          sort: 100
+
   empty: []

src/Engines/MySql/Export/Primary.php

@@ -77,7 +77,7 @@ public function __construct(
     public function go(TransportInterface $transport, EnvironmentInterface $environment)
     {
         $transport->getLogger()->notice("Exporting database from local MySql");
-        $this->output->writeln("<comment>Exporting database from local MySql");
+        $this->output->writeln("<comment>Exporting database from local MySql</comment>");
 
         $transport->getLogger()->debug(
             "Local connection string: " . str_replace(

src/Engines/MySql/Import/Primary.php

@@ -0,0 +1,129 @@
+<?php
+/**
+ * SwiftOtter_Base is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SwiftOtter_Base is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with SwiftOtter_Base. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @author Joseph Maxwell
+ * @copyright SwiftOtter Studios, 12/3/16
+ * @package default
+ **/
+
+namespace Driver\Engines\MySql\Import;
+
+use Driver\Commands\CleanupInterface;
+use Driver\Commands\CommandInterface;
+use Driver\Pipeline\Environment\EnvironmentInterface;
+use Driver\Pipeline\Transport\Status;
+use Driver\Pipeline\Transport\TransportInterface;
+use Driver\System\Configuration;
+use Driver\System\LocalConnectionLoader;
+use Driver\System\Logs\LoggerInterface;
+use Driver\System\Random;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Output\ConsoleOutput;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Primary extends Command implements CommandInterface
+{
+    /** @var LocalConnectionLoader */
+    private $localConnection;
+
+    /** @var array */
+    private $properties;
+
+    /** @var LoggerInterface */
+    private $logger;
+
+    /** @var Random */
+    private $random;
+
+    /** @var ?string */
+    private $path;
+
+    /** @var Configuration */
+    private $configuration;
+
+    /** @var ConsoleOutput */
+    private $output;
+
+    const DEFAULT_DUMP_PATH = '/tmp';
+
+    public function __construct(
+        LocalConnectionLoader $localConnection,
+        Configuration $configuration,
+        LoggerInterface $logger,
+        Random $random,
+        ConsoleOutput $output,
+        array $properties = []
+    ) {
+        $this->localConnection = $localConnection;
+        $this->properties = $properties;
+        $this->logger = $logger;
+        $this->random = $random;
+        $this->configuration = $configuration;
+        $this->output = $output;
+        return parent::__construct('import-data-from-system-primary');
+    }
+
+    public function go(TransportInterface $transport, EnvironmentInterface $environment)
+    {
+        $transport->getLogger()->notice("Import database from var/downloaded into local MySql started");
+        $this->output->writeln("<comment>Import database from var/downloaded into local MySql started</comment>");
+
+        $transport->getLogger()->debug(
+            "Local connection string: " . str_replace(
+                $this->localConnection->getPassword(),
+                '',
+                $this->assembleCommand($environment)
+            )
+        );
+        $this->output->writeln("<comment>Local connection string: </comment>" . str_replace(
+                $this->localConnection->getPassword(),
+                '',
+                $this->assembleCommand($environment)
+            )
+        );
+
+        $results = null;
+        $command = $this->assembleCommand($environment);
+
+        $results = system($command);
+
+        if ($results) {
+            $this->output->writeln('<error>Import to local MYSQL failed: ' . $results . '</error>');
+            throw new \Exception('Import to local MYSQL failed: ' . $results);
+        }
+    }
+
+    public function getProperties()
+    {
+        return $this->properties;
+    }
+
+    public function assembleCommand(EnvironmentInterface $environment)
+    {
+        return implode(' ', $this->getImportCommand($environment));
+    }
+
+    private function getImportCommand(EnvironmentInterface $environment)
+    {
+        $date = date('Y-m-d');
+        return [
+            "mysql -u \"{$this->localConnection->getUser()}\"",
+            "-h {$this->localConnection->getHost()}",
+            "-p",
+            "{$this->localConnection->getDatabase()}-".$date,
+            "<",
+            "var/downloaded/{$this->localConnection->getDatabase()}-".$date.".sql"
+        ];
+    }
+}

src/Engines/S3/Download.php

@@ -0,0 +1,158 @@
+<?php
+/**
+ * SwiftOtter_Base is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SwiftOtter_Base is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with SwiftOtter_Base. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @author Joseph Maxwell
+ * @copyright SwiftOtter Studios, 12/3/16
+ * @package default
+ **/
+
+namespace Driver\Engines\S3;
+
+use Aws\Result;
+use Aws\S3\S3Client;
+use Driver\Commands\CommandInterface;
+use Driver\Pipeline\Environment\EnvironmentInterface;
+use Driver\Pipeline\Transport\Status;
+use Driver\Pipeline\Transport\TransportInterface;
+use Driver\System\Configuration;
+use Driver\System\LocalConnectionLoader;
+use Driver\System\Logs\LoggerInterface;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Output\ConsoleOutput;
+
+class Download extends Command implements CommandInterface
+{
+    /** @var LocalConnectionLoader */
+    private $localConnection;
+    protected $configuration;
+    protected $properties;
+    /** @var LoggerInterface */
+    private $logger;
+    /** @var ConsoleOutput */
+    private $output;
+
+    public function __construct(LocalConnectionLoader $localConnection, Configuration $configuration, LoggerInterface $logger, ConsoleOutput $output, array $properties = [])
+    {
+        $this->localConnection = $localConnection;
+        $this->configuration = $configuration;
+        $this->properties = $properties;
+        $this->logger = $logger;
+        $this->output = $output;
+
+        parent::__construct('s3-download');
+    }
+
+    public function go(TransportInterface $transport, EnvironmentInterface $environment)
+    {
+        try {
+            $transport->getLogger()->notice("Beginning file download from: s3://" . $this->getBucket() . "/" . $this->getFileName($environment));
+            $this->output->writeln("<comment>Beginning file download from: s3://" . $this->getBucket() . "/" . $this->getFileName($environment) . '</comment>');
+            $date = date('Y-m-d');
+            $client = $this->getS3Client();
+            $output = $client->getObject([
+                'Bucket' => $this->getBucket(),
+                'Key' => $this->getDirectory() . $this->getFileName($environment),
+                'SourceFile' => $transport->getData($environment->getName() . '_completed_file'),
+                'ContentType' => 'application/gzip',
+                'SaveAs' => "var/downloaded/{$this->localConnection->getDatabase()}-".$date.'.sql'
+            ]);
+
+            $transport->getLogger()->notice("Downloaded file from: s3://" . $this->getBucket() . "/" . $this->getFileName($environment));
+            $this->output->writeln("<info>Downloaded file from: s3://" . $this->getBucket() . "/" . $this->getFileName($environment) . ' to project var/downloaded directory</info>');
+
+            return $transport->withNewData('s3_url', $this->getObjectUrl($output));
+        } catch (\Exception $ex) {
+            $this->logger->error('Failed getting object from S3: ' . $ex->getMessage(), [
+                $ex->getMessage(),
+                $ex->getTraceAsString()
+            ]);
+
+            $this->output->writeln('<error>Failed getting object from S3: ' . $ex->getMessage(), [
+                $ex->getMessage(),
+                $ex->getTraceAsString()
+            ] . '</error>');
+
+            return $transport->withStatus(new Status('s3-download', $ex->getMessage(), true));
+        }
+
+    }
+
+    public function getProperties()
+    {
+        return $this->properties;
+    }
+
+    protected function getObjectUrl(\Aws\Result $data)
+    {
+        return $data->get('ObjectURL');
+    }
+
+    private function getFileName(EnvironmentInterface $environment)
+    {
+        $replace = str_replace('{{environment}}', '-' . $environment->getName(), $this->getFileKey());
+        $replace = str_replace('{{date}}', date('YmdHis'), $replace);
+
+        return $replace;
+    }
+
+    private function getFileKey()
+    {
+        if ($this->compressOutput()) {
+            return $this->configuration->getNode('connections/s3/compressed-file-key');
+        } else {
+            return $this->configuration->getNode('connections/s3/uncompressed-file-key');
+        }
+    }
+
+    private function compressOutput()
+    {
+        return (bool)$this->configuration->getNode('configuration/compress-output') === true;
+    }
+
+    private function getBucket()
+    {
+        return $this->configuration->getNode('connections/s3/bucket');
+    }
+
+    private function getDirectory()
+    {
+        $directory = $this->configuration->getNode('connections/s3/directory');
+        if ($directory) {
+            $directory .= '/';
+        }
+
+        return $directory;
+    }
+
+    private function getS3Client()
+    {
+        return new S3Client($this->getAwsParameters());
+    }
+
+    private function getAwsParameters()
+    {
+        $parameters = [
+            'credentials' => [
+                'key' => $this->configuration->getNode("connections/s3/key")
+                    ?? $this->configuration->getNode("connections/aws/key"),
+                'secret' => $this->configuration->getNode("connections/s3/secret")
+                    ?? $this->configuration->getNode("connections/aws/secret")
+            ],
+            'region' => $this->configuration->getNode("connections/s3/region")
+                ?? $this->configuration->getNode("connections/aws/region"),
+            'version' => '2006-03-01'
+        ];
+        return $parameters;
+    }
+}

src/Engines/S3/Upload.php

@@ -45,7 +45,7 @@ public function __construct(Configuration $configuration, LoggerInterface $logge
         $this->properties = $properties;
         $this->logger = $logger;
         $this->output = $output;
-        
+
         parent::__construct('s3-upload');
     }
 
@@ -54,7 +54,7 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm
         try {
             $transport->getLogger()->notice("Beginning file upload to: s3://" . $this->getBucket() . "/" . $this->getFileName($environment));
             $this->output->writeln("<comment>Beginning file upload to: s3://" . $this->getBucket() . "/" . $this->getFileName($environment) . '</comment>');
-            
+
             $client = $this->getS3Client();
             $output = $client->putObject([
                 'Bucket' => $this->getBucket(),
@@ -64,15 +64,15 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm
             ]);
 
             $transport->getLogger()->notice("Uploaded file to: s3://" . $this->getBucket() . "/" . $this->getFileName($environment));
-            $this->output->writeln("<comment>Uploaded file to: s3://" . $this->getBucket() . "/" . $this->getFileName($environment) . '</comment>');
+            $this->output->writeln("<info>Uploaded file to: s3://" . $this->getBucket() . "/" . $this->getFileName($environment) . '</info>');
 
             return $transport->withNewData('s3_url', $this->getObjectUrl($output));
         } catch (\Exception $ex) {
             $this->logger->error('Failed putting object to S3: ' . $ex->getMessage(), [
                 $ex->getMessage(),
                 $ex->getTraceAsString()
             ]);
-    
+
             $this->output->writeln('<error>Failed putting object to S3: ' . $ex->getMessage(), [
                 $ex->getMessage(),
                 $ex->getTraceAsString()
@@ -150,4 +150,4 @@ private function getAwsParameters()
         ];
         return $parameters;
     }
-}
+}