some speed tweaks to make truncation quicker (#8)

* some speed tweaks to make truncation quicker

* update travis file

* lint after testing php

* fix some tests

* fix composer install in travis file

.travis.yml

@@ -23,25 +23,32 @@ matrix:
 
 before_script:
   - composer config platform.php $(php -r "echo PHP_VERSION;")
-  - travis_retry composer update --no-interaction --prefer-dist $PREFER_LOWEST
-  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - travis_retry composer update --no-ansi --no-interaction --no-suggest --no-progress --no-scripts --prefer-dist $PREFER_LOWEST
 
 script:
   - vendor/bin/phpcs -p --warning-severity=0 src/ tests/
   - vendor/bin/phpunit --coverage-clover=./tests/report/coverage.clover
-  - make lint-md
 
 after_script:
   - test -f ./tests/report/coverage.clover && (wget https://scrutinizer-ci.com/ocular.phar; php ocular.phar code-coverage:upload --format=php-clover ./tests/report/coverage.clover)
 
 stages:
   - test
+  - lint
   - name: deploy
     if: branch = master
 jobs:
   include:
+  - stage: lint
+    before_script: sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+    script: make lint-md
+    name: "Markdown"
+    after_script: skip
   - stage: deploy
+    before_script: sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
     script: make docs-build
+    name: "Deploy to GitHub Pages"
+    after_script: skip
     deploy:
       provider: pages
       local-dir: site

Dockerfile

@@ -18,7 +18,7 @@ COPY src /app/src
 COPY composer.json /app/composer.json
 COPY composer.lock /app/composer.lock
 
-RUN composer install --no-ansi --no-dev --no-interaction --no-progress --no-scripts --optimize-autoloader --prefer-dist
+RUN composer install --no-ansi --no-dev --no-interaction --no-suggest --no-progress --no-scripts --optimize-autoloader --prefer-dist
 
 FROM graze/php-alpine:7.2 AS run
 

hooks/build

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set +xe
-
 docker build --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
     --build-arg VCS_REF="$(git rev-parse --short HEAD)" \
     -t "$IMAGE_NAME" .

mkdocs.yml

@@ -9,7 +9,7 @@ docs_dir: 'docs'
 
 theme:
   name: 'material'
-  custom_dir: theme
+  custom_dir: 'docs/theme'
   favicon: assets/images/favicon.ico
   language: en
   include_sidebar: true

src/Chop/Chopper.php

@@ -59,8 +59,6 @@ public function chop(array $tables = [])
         $tableChopper = $this->factory->getChopper($this->schemaConfig->getConnection());
         $schema = $this->schemaConfig->getSchema();
 
-        foreach ($tables as $table) {
-            $tableChopper->chop($schema, $table);
-        }
+        $tableChopper->chop($schema, ...$tables);
     }
 }

src/Chop/TableChopperInterface.php

@@ -16,12 +16,12 @@
 interface TableChopperInterface
 {
     /**
-     * Take a file, and write the contents into the table within the specified schema
+     * Truncate all the provided tables in the given schema
      *
      * @param string $schema
-     * @param string $table
+     * @param string ...$tables
      *
      * @return void
      */
-    public function chop(string $schema, string $table);
+    public function chop(string $schema, string ...$tables);
 }

src/Command/ChopCommand.php

@@ -93,27 +93,12 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $schemaParser = new SchemaParser($tablePopulator, $config, $group);
         $parsedSchemas = $schemaParser->extractSchemas($schemas);
 
-        $numTables = array_sum(array_map(
-            function (ParsedSchema $schema) {
-                return count($schema->getTables());
-            },
-            $parsedSchemas
-        ));
-
-        $useGlobal = $numTables <= 10;
+        $useGlobal = count($parsedSchemas) <= 10;
 
         $globalPool = new Pool();
         $globalPool->setMaxSimultaneous($config->get(Config::CONFIG_DEFAULT_SIMULTANEOUS_PROCESSES));
 
         foreach ($parsedSchemas as $schema) {
-            $output->writeln(sprintf(
-                'Chopping down <info>%d</info> tables in <info>%s</info> schema in group <info>%s</info> from <info>%s</info>',
-                count($schema->getTables()),
-                $schema->getSchemaName(),
-                $group,
-                $schema->getPath()
-            ));
-
             if ($useGlobal) {
                 $pool = $globalPool;
             } else {

src/Db/Mysql/MysqlTableChopper.php

@@ -39,10 +39,22 @@ public function __construct(Pool $pool, ConnectionConfigInterface $connection)
 
     /**
      * @param string $schema
-     * @param string $table
+     * @param string ...$tables
      */
-    public function chop(string $schema, string $table)
+    public function chop(string $schema, string ...$tables)
     {
+        $query = sprintf(
+            'SET FOREIGN_KEY_CHECKS=0; %s; SET FOREIGN_KEY_CHECKS=1;',
+            implode(
+                '; ',
+                array_map(
+                    function (string $table) {
+                        return "TRUNCATE `{$table}`";
+                    },
+                    $tables
+                )
+            )
+        );
         $process = new Process('');
         $process->setCommandLine(
             sprintf(
@@ -51,10 +63,11 @@ public function chop(string $schema, string $table)
                 escapeshellarg($this->connection->getUser()),
                 escapeshellarg($this->connection->getPassword()),
                 escapeshellarg($schema),
-                escapeshellarg(sprintf('TRUNCATE `%s`', $table))
+                escapeshellarg($query)
             )
         );
 
-        $this->pool->add($process, ['chop', 'schema' => $schema, 'table' => $table]);
+        $displayTables = (count($tables) < 3 ? implode(', ', $tables) : count($tables));
+        $this->pool->add($process, ['chop', 'schema' => $schema, 'tables' => $displayTables]);
     }
 }

src/Db/Mysql/MysqlTableSeeder.php

@@ -57,12 +57,14 @@ public function seed(string $file, string $schema, string $table)
         $process = new Process('');
         $process->setCommandLine(
             sprintf(
-                'mysql -h%1$s -u%2$s -p%3$s --default-character-set=utf8 %4$s < %5$s',
+                '(echo %6$s; cat %5$s; echo %7$s) | mysql -h%1$s -u%2$s -p%3$s --max_allowed_packet=512M --default-character-set=utf8 %4$s',
                 escapeshellarg($this->connection->getHost()),
                 escapeshellarg($this->connection->getUser()),
                 escapeshellarg($this->connection->getPassword()),
                 escapeshellarg($schema),
-                escapeshellarg($file)
+                escapeshellarg($file),
+                escapeshellarg('SET AUTOCOMMIT=0; SET FOREIGN_KEY_CHECKS=0;'),
+                escapeshellarg('SET AUTOCOMMIT=1; SET FOREIGN_KEY_CHECKS=1;')
             )
         );
 

src/Parser/FileTablePopulator.php

@@ -64,6 +64,7 @@ function (array $a, array $b) {
                 }
             );
 
+            // remove the file extensions to get the table names
             $tables = array_map(
                 function (array $file) {
                     return pathinfo($file['path'], PATHINFO_FILENAME);
@@ -78,7 +79,6 @@ function (array $file) {
                 );
             }
 
-            // remove the file extensions to get the table names
             $parsedSchema->setTables($tables);
         }
 

tests/unit/Chop/ChopperTest.php

@@ -52,11 +52,9 @@ public function testChopperCallsChopForAllTables(array $tables = [])
 
         $factory->shouldReceive('getChopper')->with($connConfig)->andReturn($tableChopper);
 
-        foreach ($tables as $table) {
-            $tableChopper->shouldReceive('chop')
-                         ->with('schema', $table)
-                         ->once();
-        }
+        $tableChopper->shouldReceive('chop')
+                     ->with('schema', ...$tables)
+                     ->once();
 
         $chopper->chop($tables);
     }

tests/unit/Db/Mysql/MysqlTableChopperTest.php

@@ -31,7 +31,7 @@ public function testChop()
         $process = Mockery::mock('overload:' . Process::class);
 
         $process->shouldReceive('setCommandLine')
-                ->with('mysql -h\'some-host\' -u\'some-user\' -p\'some-pass\' --default-character-set=utf8 --execute=\'TRUNCATE `some-table`\' \'some-schema\'')
+                ->with('mysql -h\'some-host\' -u\'some-user\' -p\'some-pass\' --default-character-set=utf8 --execute=\'SET FOREIGN_KEY_CHECKS=0; TRUNCATE `some-table`; SET FOREIGN_KEY_CHECKS=1;\' \'some-schema\'')
                 ->once();
 
         $config = Mockery::mock(ConnectionConfigInterface::class);
@@ -47,11 +47,40 @@ public function testChop()
         $pool->shouldReceive('add')
              ->with(
                  Mockery::type(Process::class),
-                 ['chop', 'schema' => 'some-schema', 'table' => 'some-table']
+                 ['chop', 'schema' => 'some-schema', 'tables' => 'some-table']
              );
 
         $tableChopper = new MysqlTableChopper($pool, $config);
 
         $tableChopper->chop('some-schema', 'some-table');
     }
+
+    public function testChopWithMultipleTables()
+    {
+        $process = Mockery::mock('overload:' . Process::class);
+
+        $process->shouldReceive('setCommandLine')
+                ->with('mysql -h\'some-host\' -u\'some-user\' -p\'some-pass\' --default-character-set=utf8 --execute=\'SET FOREIGN_KEY_CHECKS=0; TRUNCATE `some-table`; TRUNCATE `some-table-2`; SET FOREIGN_KEY_CHECKS=1;\' \'some-schema\'')
+                ->once();
+
+        $config = Mockery::mock(ConnectionConfigInterface::class);
+        $config->shouldReceive('getHost')
+               ->andReturn('some-host');
+        $config->shouldReceive('getUser')
+               ->andReturn('some-user');
+        $config->shouldReceive('getPassword')
+               ->andReturn('some-pass');
+
+        $pool = Mockery::mock(Pool::class);
+
+        $pool->shouldReceive('add')
+             ->with(
+                 Mockery::type(Process::class),
+                 ['chop', 'schema' => 'some-schema', 'tables' => 'some-table, some-table-2']
+             );
+
+        $tableChopper = new MysqlTableChopper($pool, $config);
+
+        $tableChopper->chop('some-schema', 'some-table', 'some-table-2');
+    }
 }

tests/unit/Db/Mysql/MysqlTableSeederTest.php

@@ -32,7 +32,7 @@ public function testSeed()
         $process = Mockery::mock('overload:' . Process::class);
 
         $process->shouldReceive('setCommandLine')
-                ->with('mysql -h\'some-host\' -u\'some-user\' -p\'some-pass\' --default-character-set=utf8 \'some-schema\' < \'some-file\'')
+                ->with('(echo \'SET AUTOCOMMIT=0; SET FOREIGN_KEY_CHECKS=0;\'; cat \'some-file\'; echo \'SET AUTOCOMMIT=1; SET FOREIGN_KEY_CHECKS=1;\') | mysql -h\'some-host\' -u\'some-user\' -p\'some-pass\' --max_allowed_packet=512M --default-character-set=utf8 \'some-schema\'')
                 ->once();
 
         $config = Mockery::mock(ConnectionConfigInterface::class);